def listDataFolders(): # logger=mlog.mosaicLogging().getLogger(name=__name__) params = dict(request.get_json()) level = params.get('level', 'Data Root') if level == 'Data Root': folder = mosaic.WebServerDataLocation logger.info("/list-data-folders: " + folder) else: folder = format_path(mosaic.WebServerDataLocation + '/' + level + '/') logger.info("/list-data-folders: " + folder) folderList = [] for item in sorted(glob.glob(folder + '/*')): itemAttr = {} if os.path.isdir(item): itemAttr['name'] = os.path.relpath(item, folder) itemAttr['relpath'] = os.path.relpath( item, format_path(mosaic.WebServerDataLocation)) itemAttr['desc'] = _folderDesc(item) itemAttr['modified'] = time.strftime( '%m/%d/%Y, %I:%M %p', time.localtime(os.path.getmtime(item))) folderList.append(itemAttr) return jsonify(respondingURL='list-data-folders', level=level + '/', fileData=folderList), 200
def listDatabaseFiles(): params = dict(request.get_json()) level=params.get('level', 'Data Root') logger.info("/list-database-files: "+str(level)) if level == 'Data Root': folder=mosaic.WebServerDataLocation logger.info("/list-database-files: "+folder) else: folder=format_path(mosaic.WebServerDataLocation+'/'+level+'/') logger.info("/list-database-files: "+folder) fileList=[] for item in sorted(glob.glob(folder+'/*')): itemAttr={} if os.path.isdir(item): itemAttr['name']=os.path.relpath(item, folder) itemAttr['relpath']=os.path.relpath(item, format_path(mosaic.WebServerDataLocation) ) itemAttr['desc']=_folderDesc(item) itemAttr['modified']=time.strftime('%m/%d/%Y, %I:%M %p', time.localtime(os.path.getmtime(item))) fileList.append(itemAttr) else: if _fileExtension(item)==".sqlite": itemAttr['name']=os.path.relpath(item, folder) itemAttr['relpath']=os.path.relpath(item, format_path(mosaic.WebServerDataLocation) ) itemAttr['desc']="SQLite database, {0}".format(_fileSize(item)) itemAttr['modified']=time.strftime('%m/%d/%Y, %I:%M %p', time.localtime(os.path.getmtime(item))) fileList.append(itemAttr) return jsonify( respondingURL='list-database-files', level=level+'/', fileData=fileList ), 200
def listDataFolders(): # logger=mlog.mosaicLogging().getLogger(name=__name__) params = dict(request.get_json()) level=params.get('level', 'Data Root') if level == 'Data Root': folder=mosaic.WebServerDataLocation logger.info("/list-data-folders: "+folder) else: folder=format_path(mosaic.WebServerDataLocation+'/'+level+'/') logger.info("/list-data-folders: "+folder) folderList=[] for item in sorted(glob.glob(folder+'/*')): itemAttr={} if os.path.isdir(item): itemAttr['name']=os.path.relpath(item, folder) itemAttr['relpath']=os.path.relpath(item, format_path(mosaic.WebServerDataLocation) ) itemAttr['desc']=_folderDesc(item) itemAttr['modified']=time.strftime('%m/%d/%Y, %I:%M %p', time.localtime(os.path.getmtime(item))) folderList.append(itemAttr) return jsonify( respondingURL='list-data-folders', level=level+'/', fileData=folderList ), 200
def newAnalysis(): global gAnalysisSessions # logger=mlog.mosaicLogging().getLogger(name=__name__) try: defaultSettings = False params = dict(request.get_json()) dataPath = params.get('dataPath', None) settingsString = params.get('settingsString', None) sessionID = params.get('sessionID', None) if dataPath and not settingsString: # brand new session # print "brand new session: ", dataPath, settingsString, sessionID logger.info("/new-analysis: " + format_path(mosaic.WebServerDataLocation + '/' + dataPath)) sessionID = gAnalysisSessions.newSession() ma = mosaicAnalysis.mosaicAnalysis( format_path(mosaic.WebServerDataLocation + '/' + dataPath), sessionID) gAnalysisSessions.addDataPath( sessionID, format_path(mosaic.WebServerDataLocation + '/' + dataPath)) gAnalysisSessions.addMOSAICAnalysisObject(sessionID, ma) elif sessionID and settingsString: # update settings # print "update settings: ", dataPath, settingsString, sessionID ma = gAnalysisSessions.getSessionAttribute(sessionID, 'mosaicAnalysisObject') ma.updateSettings(settingsString) gAnalysisSessions.addSettingsString(sessionID, ma.analysisSettingsDict) elif sessionID and not settingsString: # a session ID loaded from a route # print "session id from route: ", dataPath, settingsString, sessionID ma = gAnalysisSessions.getSessionAttribute(sessionID, 'mosaicAnalysisObject') else: raise InvalidPOSTRequest('An invalid POST request was received.') return jsonify(respondingURL='new-analysis', sessionID=sessionID, **ma.setupAnalysis()), 200 except EmptyDataPipeError, err: gAnalysisSessions.pop(sessionID, None) return jsonify(respondingURL='new-analysis', errType='EmptyDataPipeError', errSummary="End of data.", errText=str(err)), 500
def runAnalysis(self): try: fname = 'eventMD-' + str( datetime.datetime.now().strftime('%Y%m%d-%H%M%S')) + '.sqlite' self.dbFile = format_path(self.dataPath + "/" + fname) # self.dbFile=self.dataPath+"/eventMD-20161208-130302.sqlite" self.processHandle = self._processHnd() self._writeSettings() self.analysisObject = sca.SingleChannelAnalysis( self.dataPath, self.trajIOHandle, None, self.partitionHandle, self.processHandle, dbFilename=fname) self.analysisObject.Run(forkProcess=True) time.sleep(3) self.analysisRunning = True except: raise
def _loadEBSState(self): path=self.analysisDataModel["DataFilesPath"] if path: ebsFile=glob.glob(format_path(str(path)+'/*_?tate.txt')) if len(ebsFile) > 0: ebsState=mosaicgui.EBSStateFileDict.EBSStateFileDict(ebsFile[0]) rfb=ebsState.pop('FB Resistance',1.0) cfb=ebsState.pop('FB Capacitance',1.0) self.qdfRfbLineEdit.setText( str(float(rfb)/1E9) ) self.qdfCfbLineEdit.setText( str(float(cfb)/1E-12) ) # More Traj viewer settings self.analysisDataModel["Rfb"]=rfb self.analysisDataModel["Cfb"]=cfb # Show QDF specific widgets self.qdfCfbLineEdit.show() self.qdfRfbLineEdit.show() self.CfbLabel.show() self.RfbLabel.show() self.CfbUnitsLabel.show() self.RfbUnitsLabel.show() else: # Hide QDF specific widgets self.qdfCfbLineEdit.hide() self.qdfRfbLineEdit.hide() self.CfbLabel.hide() self.RfbLabel.hide() self.CfbUnitsLabel.hide() self.RfbUnitsLabel.hide()
def _gaCredentialCache(): try: try: logger=mlog.mosaicLogging().getLogger(name=__name__) ga_cache=format_path(tempfile.gettempdir()+'/.ga') logger.debug(_d("Looking for GA cache {0}", ga_cache)) gaModTime = datetime.fromtimestamp(os.stat(ga_cache).st_mtime) gaExpireAge=timedelta(hours=24) gaAge=datetime.today() - gaModTime if gaAge > gaExpireAge: logger.debug(_d("GA settings cache has expired.")) ga_old=_gaSettingsDict(ga_cache) _getGASettings(ga_cache) ga_new=_gaSettingsDict(ga_cache) if ga_old["gaenable"]==False: ga_new["gaenable"]=False with open(ga_cache, "w") as ga: ga.write(json.dumps(ga_new)) else: logger.debug(_d("GA settings cache found ({0}). gaAge={1}", str(ga_cache), str(gaAge))) except: logger.debug(_d("GA settings are not cached.")) _getGASettings(ga_cache) with open(ga_cache, 'r') as ga: return json.loads(ga.read()) except BaseException as err: logger.debug(_d("Exception ignored: {0}\n{1}", repr(err), traceback.format_exc())) return
def _filename(self): """ Return a output filename that contains the data file prefix and and the block index. """ self._creategenerator() return format_path(self.outDir+'/'+next(self.fileGenerator)+'.'+self.extension)
def runAnalysis(self): try: fname='eventMD-'+str(datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))+'.sqlite' self.dbFile=format_path(self.dataPath+"/"+fname) # self.dbFile=self.dataPath+"/eventMD-20161208-130302.sqlite" self.processHandle=self._processHnd() self._writeSettings() self.analysisObject=sca.SingleChannelAnalysis( self.dataPath, self.trajIOHandle, None, self.partitionHandle, self.processHandle, dbFilename=fname ) self.analysisObject.Run(forkProcess=True) time.sleep(3) self.analysisRunning=True except: raise
def _loadEBSState(self): path = self.analysisDataModel["DataFilesPath"] if path: ebsFile = glob.glob(format_path(str(path) + '/*_?tate.txt')) if len(ebsFile) > 0: ebsState = mosaicgui.EBSStateFileDict.EBSStateFileDict( ebsFile[0]) rfb = ebsState.pop('FB Resistance', 1.0) cfb = ebsState.pop('FB Capacitance', 1.0) self.qdfRfbLineEdit.setText(str(float(rfb) / 1E9)) self.qdfCfbLineEdit.setText(str(float(cfb) / 1E-12)) # More Traj viewer settings self.analysisDataModel["Rfb"] = rfb self.analysisDataModel["Cfb"] = cfb # Show QDF specific widgets self.qdfCfbLineEdit.show() self.qdfRfbLineEdit.show() self.CfbLabel.show() self.RfbLabel.show() self.CfbUnitsLabel.show() self.RfbUnitsLabel.show() else: # Hide QDF specific widgets self.qdfCfbLineEdit.hide() self.qdfRfbLineEdit.hide() self.CfbLabel.hide() self.RfbLabel.hide() self.CfbUnitsLabel.hide() self.RfbUnitsLabel.hide()
def _initdb(self, **kwargs): """ Initialize the database tables Args: tableName name of database table. Default is 'metadata' """ self.logger=mlog.mosaicLogging().getLogger(__name__) if not hasattr(self, 'tableName'): self.tableName='metadata' if not hasattr(self, 'colNames'): raise metaMDIO.InsufficientArgumentsError("Missing arguments: 'colNames' must be supplied to initialize {0}".format(type(self).__name__)) self.logger.error("Missing arguments: 'colNames' must be supplied to initialize {0}".format(type(self).__name__)) if not hasattr(self, 'colNames_t'): raise metaMDIO.InsufficientArgumentsError("Missing arguments: 'colNames_t' must be supplied to initialize {0}".format(type(self).__name__)) self.logger.error("Missing arguments: 'colNames_t' must be supplied to initialize {0}".format(type(self).__name__)) dbTimeout=kwargs.pop('timeout', 11.0) self.logger.debug(_d("DB Timeout = {0}", dbTimeout)) self.dbFilename=format_path(self.dbPath+'/'+'eventMD-' +str(datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))+'.sqlite') self.logger.debug(_d("dbFilename {0}", self.dbFilename)) self.db = sqlite3.connect(self.dbFilename, detect_types=sqlite3.PARSE_DECLTYPES, timeout=dbTimeout) self._setuptables() self.logger.debug(_d("DB setup complete."))
def loadAnalysis(): global gAnalysisSessions try: params = dict(request.get_json()) db = params.get('databaseFile', None) databaseFile = format_path(mosaic.WebServerDataLocation + '/' + db) if not databaseFile: raise InvalidPOSTRequest( "Missing required parameter 'databaseFile'") info, settings = _dbInfo(databaseFile) dataPath = info['datPath'] sessionID = gAnalysisSessions.newSession() ma = mosaicAnalysis.mosaicAnalysis(dataPath, sessionID) ma.updateSettings(settings) # ma.setupAnalysis() gAnalysisSessions.addDatabaseFile(sessionID, databaseFile) gAnalysisSessions.addAnalysisRunningFlag(sessionID, False) gAnalysisSessions.addDataPath(sessionID, dataPath) gAnalysisSessions.addMOSAICAnalysisObject(sessionID, ma) return jsonify(respondingURL='load-analysis', sessionID=sessionID), 200 except InvalidPOSTRequest, err: return jsonify(respondingURL='load-analysis', errType='InvalidPOSTRequest', errSummary="An invalid POST request was received.", errText=str(err)), 500
def _initdb(self, **kwargs): """ Initialize the database tables Args: tableName name of database table. Default is 'metadata' """ self.logger=mlog.mosaicLogging().getLogger(__name__) if not hasattr(self, 'tableName'): self.tableName='metadata' if not hasattr(self, 'colNames'): raise metaMDIO.InsufficientArgumentsError("Missing arguments: 'colNames' must be supplied to initialize {0}".format(type(self).__name__)) self.logger.error("Missing arguments: 'colNames' must be supplied to initialize {0}".format(type(self).__name__)) if not hasattr(self, 'colNames_t'): raise metaMDIO.InsufficientArgumentsError("Missing arguments: 'colNames_t' must be supplied to initialize {0}".format(type(self).__name__)) self.logger.error("Missing arguments: 'colNames_t' must be supplied to initialize {0}".format(type(self).__name__)) dbTimeout=kwargs.pop('timeout', 11.0) self.logger.debug(_d("DB Timeout = {0}", dbTimeout)) dbfile=kwargs.get('dbFilename', '') if dbfile=='': self.dbFilename=format_path(self.dbPath+'/'+'eventMD-' +str(datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))+'.sqlite') else: self.dbFilename=dbfile self.logger.debug(_d("dbFilename {0}", self.dbFilename)) self.db = sqlite3.connect(self.dbFilename, detect_types=sqlite3.PARSE_DECLTYPES, timeout=dbTimeout) self._setuptables() self.logger.debug(_d("DB setup complete."))
def loadAnalysis(): global gAnalysisSessions try: params = dict(request.get_json()) db=params.get('databaseFile', None) databaseFile = format_path(mosaic.WebServerDataLocation+'/'+db ) if not databaseFile: raise InvalidPOSTRequest("Missing required parameter 'databaseFile'") info, settings=_dbInfo(databaseFile) dataPath=info['datPath'] sessionID=gAnalysisSessions.newSession() ma=mosaicAnalysis.mosaicAnalysis(dataPath, sessionID) ma.updateSettings(settings) # ma.setupAnalysis() gAnalysisSessions.addDatabaseFile(sessionID, databaseFile) gAnalysisSessions.addAnalysisRunningFlag(sessionID, False) gAnalysisSessions.addDataPath(sessionID, dataPath) gAnalysisSessions.addMOSAICAnalysisObject(sessionID, ma) return jsonify(respondingURL='load-analysis', sessionID=sessionID ), 200 except InvalidPOSTRequest, err: return jsonify( respondingURL='load-analysis', errType='InvalidPOSTRequest', errSummary="An invalid POST request was received.", errText=str(err) ), 500
def __init__(self, **kwargs): """ """ self.CHUNKSIZE = 10000 self.dataGenerator = None # start by setting all passed keyword arguments as class attributes for (k, v) in kwargs.iteritems(): setattr(self, k, v) # Check if the passed arguments are sane if hasattr(self, 'dirname') and hasattr(self, 'fnames'): raise IncompatibleArgumentsError( "Incompatible arguments: expect either 'dirname' or 'fnames' when initializing class {0}." .format(type(self).__name__)) # Check for the filter arg if not hasattr(self, 'filter'): self.filter = '*' if hasattr(self, 'fnames'): # set fnames here. self.dataFiles = self.fnames delattr(self, 'fnames') else: try: if hasattr(self, 'dirname') and hasattr(self, 'nfiles'): # N files from a directory self.dataFiles = glob.glob( format_path( str(self.dirname) + "/" + str(self.filter)))[:int(self.nfiles)] delattr(self, 'dirname') delattr(self, 'nfiles') elif hasattr(self, 'dirname'): # all files from a directory self.dataFiles = glob.glob( format_path( str(self.dirname) + "/" + str(self.filter))) delattr(self, 'dirname') else: raise IncompatibleArgumentsError( "Missing arguments: 'dirname' or 'fnames' must be supplied to initialize {0}" .format(type(self).__name__)) except AttributeError, err: raise IncompatibleArgumentsError(err)
def exportToCSV(self, query): """ Export database records that match the specified query to a CSV flat file. """ csvfile=format_path( self.dbFile.split('.')[0]+'.csv' ) df=pandas.DataFrame(self.queryDB(query), columns=self._col_names(query, self.db.cursor(), self.tableName)) df.to_csv( csvfile ) self.logger.debug(_d("{0}", csvfile))
def dropEvent( self, event ): data = event.mimeData() urls = data.urls() if ( urls and urls[0].scheme() == 'file' ): # for some reason, this doubles up the intro slash filepath = str( format_path( urls[0].path()) ) if filepath.endswith('.sqlite'): self.setText(filepath)
def dropEvent(self, event): data = event.mimeData() urls = data.urls() if (urls and urls[0].scheme() == 'file'): # for some reason, this doubles up the intro slash filepath = str(format_path(urls[0].path())) if filepath.endswith('.sqlite'): self.setText(filepath)
def _uuid(): uuidfile=format_path(tempfile.gettempdir()+'/.mosaicuuid') try: with open (uuidfile, "r") as u: return u.read() except: uuidgen=str(uuid.uuid4()) with open(uuidfile, "w") as uw: uw.write(uuidgen) return uuidgen
def _uuid(): uuidfile=format_path(expanduser('~')+"/.mosaicuuid") try: with open (uuidfile, "r") as u: return u.read() except: uuidgen=str(uuid.uuid4()) with open(uuidfile, "w") as uw: uw.write(uuidgen) return uuidgen
def _uuid(): uuidfile = format_path(expanduser('~') + "/.mosaicuuid") try: with open(uuidfile, "r") as u: return u.read() except: uuidgen = str(uuid.uuid4()) with open(uuidfile, "w") as uw: uw.write(uuidgen) return uuidgen
def _uuid(): uuidfile = format_path(tempfile.gettempdir() + '/.mosaicuuid') try: with open(uuidfile, "r") as u: return u.read() except: uuidgen = str(uuid.uuid4()) with open(uuidfile, "w") as uw: uw.write(uuidgen) return uuidgen
def listDatabaseFiles(): params = dict(request.get_json()) level = params.get('level', 'Data Root') logger.info("/list-database-files: " + str(level)) if level == 'Data Root': folder = mosaic.WebServerDataLocation logger.info("/list-database-files: " + folder) else: folder = format_path(mosaic.WebServerDataLocation + '/' + level + '/') logger.info("/list-database-files: " + folder) fileList = [] for item in sorted(glob.glob(folder + '/*')): itemAttr = {} if os.path.isdir(item): itemAttr['name'] = os.path.relpath(item, folder) itemAttr['relpath'] = os.path.relpath( item, format_path(mosaic.WebServerDataLocation)) itemAttr['desc'] = _folderDesc(item) itemAttr['modified'] = time.strftime( '%m/%d/%Y, %I:%M %p', time.localtime(os.path.getmtime(item))) fileList.append(itemAttr) else: if _fileExtension(item) == ".sqlite": itemAttr['name'] = os.path.relpath(item, folder) itemAttr['relpath'] = os.path.relpath( item, format_path(mosaic.WebServerDataLocation)) itemAttr['desc'] = "SQLite database, {0}".format( _fileSize(item)) itemAttr['modified'] = time.strftime( '%m/%d/%Y, %I:%M %p', time.localtime(os.path.getmtime(item))) fileList.append(itemAttr) return jsonify(respondingURL='list-database-files', level=level + '/', fileData=fileList), 200
def OnAggregateUsage(self): try: ga_cache = format_path(tempfile.gettempdir() + '/.ga') with open(ga_cache, "r") as garead: gac = json.load(garead) gac["gaenable"] = str(self.actionAggregate_Usage.isChecked()) with open(ga_cache, "w") as gawrite: json.dump(gac, gawrite, indent=4, sort_keys=True) except: pass
def newAnalysis(): global gAnalysisSessions # logger=mlog.mosaicLogging().getLogger(name=__name__) try: defaultSettings=False params = dict(request.get_json()) dataPath = params.get('dataPath', None) settingsString = params.get('settingsString', None) sessionID=params.get('sessionID', None) if dataPath and not settingsString: # brand new session # print "brand new session: ", dataPath, settingsString, sessionID logger.info("/new-analysis: "+format_path(mosaic.WebServerDataLocation+'/'+dataPath)) sessionID=gAnalysisSessions.newSession() ma=mosaicAnalysis.mosaicAnalysis( format_path(mosaic.WebServerDataLocation+'/'+dataPath), sessionID) gAnalysisSessions.addDataPath(sessionID, format_path(mosaic.WebServerDataLocation+'/'+dataPath) ) gAnalysisSessions.addMOSAICAnalysisObject(sessionID, ma) elif sessionID and settingsString: # update settings # print "update settings: ", dataPath, settingsString, sessionID ma=gAnalysisSessions.getSessionAttribute(sessionID, 'mosaicAnalysisObject') ma.updateSettings(settingsString) gAnalysisSessions.addSettingsString(sessionID, ma.analysisSettingsDict) elif sessionID and not settingsString: # a session ID loaded from a route # print "session id from route: ", dataPath, settingsString, sessionID ma=gAnalysisSessions.getSessionAttribute(sessionID, 'mosaicAnalysisObject') else: raise InvalidPOSTRequest('An invalid POST request was received.') return jsonify(respondingURL='new-analysis', sessionID=sessionID, **ma.setupAnalysis() ), 200 except EmptyDataPipeError, err: gAnalysisSessions.pop(sessionID, None) return jsonify( respondingURL='new-analysis', errType='EmptyDataPipeError', errSummary="End of data.", errText=str(err) ), 500
def initialization(): ga_cache=format_path(tempfile.gettempdir()+'/.ga') params = dict(request.get_json()) appAnalytics=params.get("appAnalytics", -1) gac=ga._gaCredentialCache() if appAnalytics!=-1: gac["gaenable"]=str(appAnalytics) with open(ga_cache, "w") as g: g.write(json.dumps(gac)) return jsonify( respondingURL="initialization", appAnalytics=0, showAnalyticsOptions=0, serverMode=mosaic.WebServerMode), 200
def __init__(self, **kwargs): """ """ self.CHUNKSIZE=10000 self.dataGenerator=None # start by setting all passed keyword arguments as class attributes for (k,v) in kwargs.iteritems(): setattr(self, k, v) # Check if the passed arguments are sane if hasattr(self, 'dirname') and hasattr(self, 'fnames'): raise IncompatibleArgumentsError("Incompatible arguments: expect either 'dirname' or 'fnames' when initializing class {0}.".format(type(self).__name__)) # Check for the filter arg if not hasattr(self, 'filter'): self.filter='*' if hasattr(self, 'fnames'): # set fnames here. self.dataFiles=self.fnames delattr(self, 'fnames') else: try: if hasattr(self, 'dirname') and hasattr(self,'nfiles'): # N files from a directory self.dataFiles=glob.glob(format_path(str(self.dirname)+"/"+str(self.filter)))[:int(self.nfiles)] delattr(self, 'dirname') delattr(self, 'nfiles') elif hasattr(self, 'dirname'): # all files from a directory self.dataFiles=glob.glob(format_path(str(self.dirname)+"/"+str(self.filter))) delattr(self, 'dirname') else: raise IncompatibleArgumentsError("Missing arguments: 'dirname' or 'fnames' must be supplied to initialize {0}".format(type(self).__name__)) except AttributeError, err: raise IncompatibleArgumentsError(err)
def __init__(self, trajDataObj, eventProcHnd, eventPartitionSettings, eventProcSettings, settingsString): """ Initialize a new event segment object """ # Required arguments self.trajDataObj=trajDataObj self.eventProcHnd=eventProcHnd self.settingsDict = eventPartitionSettings self.eventProcSettingsDict = eventProcSettings try: self.writeEventTS=int(self.settingsDict.pop("writeEventTS",1)) self.parallelProc=int(self.settingsDict.pop("parallelProc",1)) self.reserveNCPU=int(self.settingsDict.pop("reserveNCPU",2)) except ValueError as err: raise commonExceptions.SettingsTypeError( err ) sys.stdout.flush() self.logFileHnd=open(format_path(self.trajDataObj.datPath+'/eventProcessing.log'),'w') self.tEventProcObj=self.eventProcHnd([], self.trajDataObj.FsHz, eventstart=0,eventend=0, baselinestats=[ 0,0,0 ], algosettingsdict=self.eventProcSettingsDict.copy(), savets=False, absdatidx=0, datafileHnd=None ) self.mdioDBHnd=sqlite3MDIO.sqlite3MDIO() self.mdioDBHnd.initDB( dbPath=self.trajDataObj.datPath, tableName='metadata', colNames=(self.tEventProcObj.mdHeadings())+['TimeSeries'], colNames_t=(self.tEventProcObj.mdHeadingDataType())+['REAL_LIST'] ) self.mdioDBHnd.writeSettings(settingsString) if self.trajDataObj.dataFilter: fstring=type(self.trajDataObj.dataFilterObj).__name__ else: fstring='None' self.mdioDBHnd.writeAnalysisInfo([ self.trajDataObj.datPath, self.trajDataObj.fileFormat, type(self).__name__, type(self.tEventProcObj).__name__, fstring, ]) if self.parallelProc: self._setupparallel() self._init(trajDataObj, eventProcHnd, eventPartitionSettings, eventProcSettings)
def initialization(): ga_cache = format_path(tempfile.gettempdir() + '/.ga') params = dict(request.get_json()) appAnalytics = params.get("appAnalytics", -1) gac = ga._gaCredentialCache() if appAnalytics != -1: gac["gaenable"] = str(appAnalytics) with open(ga_cache, "w") as g: g.write(json.dumps(gac)) return jsonify(respondingURL="initialization", appAnalytics=0, showAnalyticsOptions=0, serverMode=mosaic.WebServerMode), 200
def _initdb(self, **kwargs): """ Initialize the database tables Args: tableName name of database table. Default is 'metadata' """ if not hasattr(self, 'tableName'): self.tableName='metadata' if not hasattr(self, 'colNames'): raise metaMDIO.InsufficientArgumentsError("Missing arguments: 'colNames' must be supplied to initialize {0}".format(type(self).__name__)) if not hasattr(self, 'colNames_t'): raise metaMDIO.InsufficientArgumentsError("Missing arguments: 'colNames_t' must be supplied to initialize {0}".format(type(self).__name__)) dbTimeout=kwargs.pop('timeout', 11.0) self.dbFilename=format_path(self.dbPath+'/'+'eventMD-' +str(datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))+'.sqlite') self.db = sqlite3.connect(self.dbFilename, detect_types=sqlite3.PARSE_DECLTYPES, timeout=dbTimeout) self._setuptables()
def analysisStatistics(self): statsDict = {} s = self._eventStats() statsDict['fractionNormal'] = s[0] statsDict['fractionWarn'] = s[1] statsDict['fractionError'] = s[2] statsDict['nTotal'] = s[3] c = self._caprate() statsDict['captureRateMean'] = c[0] statsDict['captureRateSigma'] = c[1] t = self._timePerEvent() statsDict['processTimePerEventMean'] = t[0] statsDict['processTimePerEventSigma'] = t[1] o = self._openChanCurrent() statsDict['openChannelCurrentMean'] = o[0] statsDict['openChannelCurrentSigma'] = o[1] statsDict['analysisProgressPercent'] = self._analysisProgress() dbHnd = sqlite.sqlite3MDIO() dbHnd.openDB(self.analysisDB) analysisInfo = dbHnd.readAnalysisInfo() procName = analysisStatistics.processingAlgorithmName partName = analysisStatistics.partitionAlgorithmName statsDict['partitionAlgorithm'] = partName[ analysisInfo['partitionAlgorithm']] statsDict['processingAlgorithm'] = procName[ analysisInfo['processingAlgorithm']] statsDict['FskHz'] = analysisInfo['FsHz'] / 1000. statsDict['dataType'] = analysisInfo['dataType'] statsDict['datPath'] = format_path((analysisInfo['datPath'].replace( str(WebServerDataLocation), "<Data Root>/")).replace('//', '/')) statsDict['timeseriesSaved'] = self._timeseriesSaved() return statsDict
def analysisStatistics(self): statsDict={} s=self._eventStats() statsDict['fractionNormal']=s[0] statsDict['fractionWarn']=s[1] statsDict['fractionError']=s[2] statsDict['nTotal']=s[3] c=self._caprate() statsDict['captureRateMean']=c[0] statsDict['captureRateSigma']=c[1] t=self._timePerEvent() statsDict['processTimePerEventMean']=t[0] statsDict['processTimePerEventSigma']=t[1] o=self._openChanCurrent() statsDict['openChannelCurrentMean']=o[0] statsDict['openChannelCurrentSigma']=o[1] statsDict['analysisProgressPercent']=self._analysisProgress() dbHnd=sqlite.sqlite3MDIO() dbHnd.openDB(self.analysisDB) analysisInfo=dbHnd.readAnalysisInfo() procName=analysisStatistics.processingAlgorithmName partName=analysisStatistics.partitionAlgorithmName statsDict['partitionAlgorithm']=partName[analysisInfo['partitionAlgorithm']] statsDict['processingAlgorithm']=procName[analysisInfo['processingAlgorithm']] statsDict['FskHz']=analysisInfo['FsHz']/1000. statsDict['dataType']=analysisInfo['dataType'] statsDict['datPath']=format_path((analysisInfo['datPath'].replace(str(WebServerDataLocation), "<Data Root>/")).replace('//', '/')) statsDict['timeseriesSaved']=self._timeseriesSaved() return statsDict
def _gaCredentialCache(): try: try: logger = mlog.mosaicLogging().getLogger(name=__name__) ga_cache = format_path(tempfile.gettempdir() + '/.ga') logger.debug(_d("Looking for GA cache {0}", ga_cache)) gaModTime = datetime.fromtimestamp(os.stat(ga_cache).st_mtime) gaExpireAge = timedelta(hours=24) gaAge = datetime.today() - gaModTime if gaAge > gaExpireAge: logger.debug(_d("GA settings cache has expired.")) ga_old = _gaSettingsDict(ga_cache) _getGASettings(ga_cache) ga_new = _gaSettingsDict(ga_cache) if ga_old["gaenable"] == False: ga_new["gaenable"] = False with open(ga_cache, "w") as ga: ga.write(json.dumps(ga_new)) else: logger.debug( _d("GA settings cache found ({0}). gaAge={1}", str(ga_cache), str(gaAge))) except: logger.debug(_d("GA settings are not cached.")) _getGASettings(ga_cache) with open(ga_cache, 'r') as ga: return json.loads(ga.read()) except BaseException as err: logger.debug( _d("Exception ignored: {0}\n{1}", repr(err), traceback.format_exc())) return
def _gaCredentialCache(): logger = mlog.mosaicLogging().getLogger(name=__name__) # ga_cache=resource_path("mosaic/utilities/.ga") ga_cache = format_path(tempfile.gettempdir() + '/.ga') logger.debug(_d("Looking for GA cache {0}", ga_cache)) try: gaModTime = datetime.fromtimestamp(os.stat(ga_cache).st_mtime) gaExpireAge = timedelta(hours=24) gaAge = datetime.today() - gaModTime if gaAge > gaExpireAge: logger.debug(_d("GA settings cache has expired.")) _getGASettings(ga_cache) else: logger.debug(_d("GA settings cache found. gaAge={0}", gaAge)) except: logger.debug(_d("GA settings are not cached.")) _getGASettings(ga_cache) with open(ga_cache, 'r') as ga: return json.loads(ga.read())
class mosaicLogging(object): """ A custom logging class that uses the Python logging facility. Logs are automatically saved to a metaMDIO instance, and to a file log when DeveloperMode is active. """ __metaclass__ = metaSingleton _loggers = {} log = logging.getLogger() log.setLevel(logging.DEBUG) formatstr = MessageFormatter( "%(asctime)-8s %(levelname)-8s %(name)-12s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") # Rotating File Handler try: logdir = mosaic.LogLocation log.info("Logs will be saved to: {0}".format(logdir)) except AttributeError, err: if sys.platform.startswith('darwin'): logdir = format_path( os.path.expanduser('~') + "/Library/Logs/MOSAIC") if not os.path.exists(logdir): os.mkdir(logdir) elif sys.platform.startswith('linux'): if os.getuid() == 0: logdir = "/var/log/" else: log.info( "MOSAIC log will be saved to ~/mosaic.log. Run MOSAIC with sudo to save logs to '/var/log/." ) logdir = os.path.expanduser("~") else: logdir = os.path.expanduser("~")
logdir = format_path( os.path.expanduser('~') + "/Library/Logs/MOSAIC") if not os.path.exists(logdir): os.mkdir(logdir) elif sys.platform.startswith('linux'): if os.getuid() == 0: logdir = "/var/log/" else: log.info( "MOSAIC log will be saved to ~/mosaic.log. Run MOSAIC with sudo to save logs to '/var/log/." ) logdir = os.path.expanduser("~") else: logdir = os.path.expanduser("~") logname = format_path(logdir + "/mosaic.log") rfh = logging.handlers.RotatingFileHandler(filename=logname, maxBytes=mosaic.LogSizeBytes, backupCount=5) rfh.setFormatter(formatstr) if mosaic.DeveloperMode: rfh.setLevel(logging.DEBUG) else: rfh.setLevel(logging.INFO) log.addHandler(rfh) sh = None def __init__(self, *args, **kwargs):
def _updateControls(self): self.updateDialogs=False model=self.analysisDataModel datidx= { "QDF" : self.datTypeComboBox.findText("QDF"), "ABF" : self.datTypeComboBox.findText("ABF"), "BIN" : self.datTypeComboBox.findText("BIN") } path=model["DataFilesPath"] if len(glob.glob(format_path( str(path)+'/*qdf') )) > 0: self.datTypeComboBox.setCurrentIndex( datidx["QDF"] ) model["filter"]="*.qdf" elif len(glob.glob( format_path(str(path)+'/*abf') )) > 0: self.datTypeComboBox.setCurrentIndex( datidx["ABF"] ) model["filter"]="*.abf" elif len(glob.glob( format_path(str(path)+'/*bin') )) > 0: self.datTypeComboBox.setCurrentIndex( datidx["BIN"] ) model["filter"]="*.bin" elif len(glob.glob( format_path(str(path)+'/*dat') )) > 0: self.datTypeComboBox.setCurrentIndex( datidx["BIN"] ) model["filter"]="*.dat" # store the data type in the trajviewer data struct model["DataFilesType"] = str(self.datTypeComboBox.currentText()) self.startIndexLineEdit.setText(str(model["start"])) if model["end"]==-1: self.endIndexLineEdit.setText("") else: self.endIndexLineEdit.setText(str(model["end"])) self.dcOffsetDoubleSpinBox.setValue(model["dcOffset"]) if float(model["meanOpenCurr"]) == -1. or float(model["sdOpenCurr"]) == -1.: self.baselineAutoCheckBox.setChecked(True) self.OnBaselineAutoCheckbox(True) # Manually disable baseline mean and SD controls self.baselineMeanLineEdit.setText("") self.baselineSDLineEdit.setText("") self.baselineMeanLineEdit.setPlaceholderText("<auto>") self.baselineSDLineEdit.setPlaceholderText("<auto>") self.baselineMeanLineEdit.setEnabled(False) self.baselineSDLineEdit.setEnabled(False) else: # Populate baseline parameters self.baselineAutoCheckBox.setChecked(False) self.OnBaselineAutoCheckbox(False) self.baselineMeanLineEdit.setText(str(model["meanOpenCurr"])) self.baselineSDLineEdit.setText(str(model["sdOpenCurr"])) self.baselineBlockSizeDoubleSpinBox.setValue(float(model["blockSizeSec"])) # Manually enable baseline mean and SD controls self.baselineMeanLineEdit.setEnabled(True) self.baselineSDLineEdit.setEnabled(True) # Populate EventSegment parameters self._setThreshold(float(model["meanOpenCurr"]), float(model["sdOpenCurr"]), float(model["eventThreshold"])) # Populate misc parameters self.writeEventsCheckBox.setChecked(int(model["writeEventTS"])) # Populate plot widgets self.plotEventFitsCheckBox.setEnabled(int(model["writeEventTS"])) # check if parallel is available try: import zmq self.parallelCheckBox.setChecked(int(model["parallelProc"])) self.parallelCoresSpinBox.setValue(multiprocessing.cpu_count()-int(model["reserveNCPU"])) except ImportError: self.parallelCheckBox.hide() self.parallelCoresSpinBox.hide() self.parallelCoresLabel.hide() procidx= {} for v in self.analysisDataModel.eventProcessingAlgoKeys.values(): procidx[v]=self.processingAlgorithmComboBox.findText(v) self.processingAlgorithmComboBox.setCurrentIndex( procidx[self.analysisDataModel["ProcessingAlgorithm"]] ) # If an advanced mode dialog exists, update its settings if self.advancedSettingsDialog: if self.dataFilterDenoise: fltr=self.analysisDataModel["FilterAlgorithm"] else: fltr=None self.advancedSettingsDialog.updateSettingsString( model.GenerateSettingsView( eventPartitionAlgo=str(self.partitionAlgorithmComboBox.currentText()), eventProcessingAlgo=str(self.processingAlgorithmComboBox.currentText()), dataFilterAlgo=fltr ) ) # If the trajviewer is initialized, update the denoising settings if self.trajViewerWindow: self.trajViewerWindow.waveletLevelSpinBox.setValue(int(self.analysisDataModel["level"])) self.updateDialogs=True
def _getdbfiles(self): path=self.analysisDataModel["DataFilesPath"] return glob.glob(format_path(path+'/*sqlite'))
def _buildFileList(self, dirname, filter): flist = set(glob.glob(format_path(dirname + "/" + filter))) for ignorefilter in ignorelist: flist = flist - set( glob.glob(format_path(dirname + "/" + ignorefilter))) return list(flist)
def _buildFileList(self, dirname, filter): flist=set(glob.glob(format_path(dirname+"/"+filter))) for ignorefilter in ignorelist: flist=flist-set(glob.glob(format_path(dirname+"/"+ignorefilter))) return list(flist)
def _updateControls(self): self.updateDialogs = False model = self.analysisDataModel datidx = { "QDF": self.datTypeComboBox.findText("QDF"), "ABF": self.datTypeComboBox.findText("ABF"), "BIN": self.datTypeComboBox.findText("BIN") } path = model["DataFilesPath"] if len(glob.glob(format_path(str(path) + '/*qdf'))) > 0: self.datTypeComboBox.setCurrentIndex(datidx["QDF"]) model["filter"] = "*.qdf" elif len(glob.glob(format_path(str(path) + '/*abf'))) > 0: self.datTypeComboBox.setCurrentIndex(datidx["ABF"]) model["filter"] = "*.abf" elif len(glob.glob(format_path(str(path) + '/*bin'))) > 0: self.datTypeComboBox.setCurrentIndex(datidx["BIN"]) model["filter"] = "*.bin" elif len(glob.glob(format_path(str(path) + '/*dat'))) > 0: self.datTypeComboBox.setCurrentIndex(datidx["BIN"]) model["filter"] = "*.dat" # store the data type in the trajviewer data struct model["DataFilesType"] = str(self.datTypeComboBox.currentText()) self.startIndexLineEdit.setText(str(model["start"])) if model["end"] == -1: self.endIndexLineEdit.setText("") else: self.endIndexLineEdit.setText(str(model["end"])) self.dcOffsetDoubleSpinBox.setValue(model["dcOffset"]) if float(model["meanOpenCurr"]) == -1. or float( model["sdOpenCurr"]) == -1.: self.baselineAutoCheckBox.setChecked(True) self.OnBaselineAutoCheckbox(True) # Manually disable baseline mean and SD controls self.baselineMeanLineEdit.setText("") self.baselineSDLineEdit.setText("") self.baselineMeanLineEdit.setPlaceholderText("<auto>") self.baselineSDLineEdit.setPlaceholderText("<auto>") self.baselineMeanLineEdit.setEnabled(False) self.baselineSDLineEdit.setEnabled(False) else: # Populate baseline parameters self.baselineAutoCheckBox.setChecked(False) self.OnBaselineAutoCheckbox(False) self.baselineMeanLineEdit.setText(str(model["meanOpenCurr"])) self.baselineSDLineEdit.setText(str(model["sdOpenCurr"])) self.baselineBlockSizeDoubleSpinBox.setValue( float(model["blockSizeSec"])) # Manually enable baseline mean and SD controls self.baselineMeanLineEdit.setEnabled(True) self.baselineSDLineEdit.setEnabled(True) # Populate EventSegment parameters self._setThreshold(float(model["meanOpenCurr"]), float(model["sdOpenCurr"]), float(model["eventThreshold"])) # Populate misc parameters self.writeEventsCheckBox.setChecked(int(model["writeEventTS"])) # Populate plot widgets self.plotEventFitsCheckBox.setEnabled(int(model["writeEventTS"])) # check if parallel is available try: import zmq self.parallelCheckBox.setChecked(int(model["parallelProc"])) self.parallelCoresSpinBox.setValue(multiprocessing.cpu_count() - int(model["reserveNCPU"])) except ImportError: self.parallelCheckBox.hide() self.parallelCoresSpinBox.hide() self.parallelCoresLabel.hide() procidx = {} for v in self.analysisDataModel.eventProcessingAlgoKeys.values(): procidx[v] = self.processingAlgorithmComboBox.findText(v) self.processingAlgorithmComboBox.setCurrentIndex( procidx[self.analysisDataModel["ProcessingAlgorithm"]]) # If an advanced mode dialog exists, update its settings if self.advancedSettingsDialog: if self.dataFilterDenoise: fltr = self.analysisDataModel["FilterAlgorithm"] else: fltr = None self.advancedSettingsDialog.updateSettingsString( model.GenerateSettingsView( eventPartitionAlgo=str( self.partitionAlgorithmComboBox.currentText()), eventProcessingAlgo=str( self.processingAlgorithmComboBox.currentText()), dataFilterAlgo=fltr)) # If the trajviewer is initialized, update the denoising settings if self.trajViewerWindow: self.trajViewerWindow.waveletLevelSpinBox.setValue( int(self.analysisDataModel["level"])) self.updateDialogs = True
def __init__(self, trajDataObj, eventProcHnd, eventPartitionSettings, eventProcSettings, settingsString): """ Initialize a new event segment object """ # Required arguments self.trajDataObj = trajDataObj self.eventProcHnd = eventProcHnd self.settingsDict = eventPartitionSettings self.eventProcSettingsDict = eventProcSettings try: self.writeEventTS = int(self.settingsDict.pop("writeEventTS", 1)) self.parallelProc = int(self.settingsDict.pop("parallelProc", 1)) self.reserveNCPU = int(self.settingsDict.pop("reserveNCPU", 2)) except ValueError as err: raise commonExceptions.SettingsTypeError(err) sys.stdout.flush() self.logFileHnd = open( format_path(self.trajDataObj.datPath + '/eventProcessing.log'), 'w') self.tEventProcObj = self.eventProcHnd( [], self.trajDataObj.FsHz, eventstart=0, eventend=0, baselinestats=[0, 0, 0], algosettingsdict=self.eventProcSettingsDict.copy(), savets=False, absdatidx=0, datafileHnd=None) self.mdioDBHnd = sqlite3MDIO.sqlite3MDIO() self.mdioDBHnd.initDB( dbPath=self.trajDataObj.datPath, tableName='metadata', colNames=(self.tEventProcObj.mdHeadings()) + ['TimeSeries'], colNames_t=(self.tEventProcObj.mdHeadingDataType()) + ['REAL_LIST']) self.mdioDBHnd.writeSettings(settingsString) if self.trajDataObj.dataFilter: fstring = type(self.trajDataObj.dataFilterObj).__name__ else: fstring = 'None' self.mdioDBHnd.writeAnalysisInfo([ self.trajDataObj.datPath, self.trajDataObj.fileFormat, type(self).__name__, type(self.tEventProcObj).__name__, fstring, ]) if self.parallelProc: self._setupparallel() self._init(trajDataObj, eventProcHnd, eventPartitionSettings, eventProcSettings)
def _updateControls(self): self.updateDialogs = False model = self.analysisDataModel datidx = { "QDF": self.datTypeComboBox.findText("QDF"), "ABF": self.datTypeComboBox.findText("ABF"), "BIN": self.datTypeComboBox.findText("BIN"), "TSV": self.datTypeComboBox.findText("TSV"), "LOG": self.datTypeComboBox.findText("LOG") } path = model["DataFilesPath"] if len(glob.glob(format_path(str(path) + '/*qdf'))) > 0: self.datTypeComboBox.setCurrentIndex(datidx["QDF"]) model["filter"] = "*.qdf" elif len(glob.glob(format_path(str(path) + '/*abf'))) > 0: self.datTypeComboBox.setCurrentIndex(datidx["ABF"]) model["filter"] = "*.abf" elif len(glob.glob(format_path(str(path) + '/*bin'))) > 0: self.datTypeComboBox.setCurrentIndex(datidx["BIN"]) model["filter"] = "*.bin" elif len(glob.glob(format_path(str(path) + '/*dat'))) > 0: self.datTypeComboBox.setCurrentIndex(datidx["BIN"]) model["filter"] = "*.dat" elif len(glob.glob(format_path(str(path) + '/*tsv'))) > 0: self.datTypeComboBox.setCurrentIndex(datidx["TSV"]) model["filter"] = "*.tsv" elif len(glob.glob(format_path(str(path) + '/*txt'))) > 0 and len( glob.glob(format_path(str(path) + '/*log'))) == 0: self.datTypeComboBox.setCurrentIndex(datidx["TSV"]) model["filter"] = "*.txt" elif len(glob.glob(format_path(str(path) + '/*log'))) > 0: self.datTypeComboBox.setCurrentIndex(datidx["LOG"]) model["filter"] = "*.log" # store the data type in the trajviewer data struct model["DataFilesType"] = str(self.datTypeComboBox.currentText()) self.startIndexLineEdit.setText(str(model["start"])) if model["end"] == -1: self.endIndexLineEdit.setText("") else: self.endIndexLineEdit.setText(str(model["end"])) self.dcOffsetDoubleSpinBox.setValue(model["dcOffset"]) if float(model["meanOpenCurr"]) == -1. or float( model["sdOpenCurr"]) == -1.: self.baselineAutoCheckBox.setChecked(True) self.OnBaselineAutoCheckbox(True) # Manually disable baseline mean and SD controls self.baselineMeanLineEdit.setText("") self.baselineSDLineEdit.setText("") self.baselineMeanLineEdit.setPlaceholderText("<auto>") self.baselineSDLineEdit.setPlaceholderText("<auto>") self.baselineMeanLineEdit.setEnabled(False) self.baselineSDLineEdit.setEnabled(False) else: # Populate baseline parameters self.baselineAutoCheckBox.setChecked(False) self.OnBaselineAutoCheckbox(False) self.baselineMeanLineEdit.setText(str(model["meanOpenCurr"])) self.baselineSDLineEdit.setText(str(model["sdOpenCurr"])) self.baselineBlockSizeDoubleSpinBox.setValue( float(model["blockSizeSec"])) # Manually enable baseline mean and SD controls self.baselineMeanLineEdit.setEnabled(True) self.baselineSDLineEdit.setEnabled(True) # Populate EventSegment parameters self._setThreshold(float(model["meanOpenCurr"]), float(model["sdOpenCurr"]), float(model["eventThreshold"])) # Populate misc parameters self.writeEventsCheckBox.setChecked(int(model["writeEventTS"])) # Populate plot widgets self.plotEventFitsCheckBox.setEnabled(int(model["writeEventTS"])) # check if parallel is available # try: # import zmq # self.parallelCheckBox.setChecked(int(model["parallelProc"])) # self.parallelCoresSpinBox.setValue(multiprocessing.cpu_count()-int(model["reserveNCPU"])) # except ImportError: # self.parallelCheckBox.hide() # self.parallelCoresSpinBox.hide() # self.parallelCoresLabel.hide() self.parallelCheckBox.hide() self.parallelCoresSpinBox.hide() self.parallelCoresLabel.hide() procidx = {} for v in self.analysisDataModel.eventProcessingAlgoKeys.keys(): procidx[v] = self.processingAlgorithmComboBox.findText(v) self.processingAlgorithmComboBox.setCurrentIndex( procidx[self.analysisDataModel.EventProcessingAlgorithmLabel()]) # If an advanced mode dialog exists, update its settings if self.advancedSettingsDialog: if self.dataFilterDenoise: fltr = self.analysisDataModel["FilterAlgorithm"] else: fltr = None self.advancedSettingsDialog.updateSettingsString( model.GenerateSettingsView( eventPartitionAlgo=str( self.partitionAlgorithmComboBox.currentText()), eventProcessingAlgo=str( self.processingAlgorithmComboBox.currentText()), dataFilterAlgo=fltr)) # If the trajviewer is initialized, update the denoising settings if self.trajViewerWindow: self.trajViewerWindow.waveletLevelSpinBox.setValue( int(self.analysisDataModel["level"])) # Hide Rfb and Cfb for QDF files [ control.hide() for control in [ self.RfbLabel, self.qdfRfbLineEdit, self.RfbUnitsLabel, self.CfbLabel, self.qdfCfbLineEdit, self.CfbUnitsLabel ] ] # Set ga toggle. try: ga_cache = format_path(tempfile.gettempdir() + '/.ga') with open(ga_cache, "r") as garead: gac = json.load(garead) if eval(gac["gauimode"]): self.actionAggregate_Usage.setVisible(True) else: self.actionAggregate_Usage.setVisible(False) if eval(gac["gaenable"]): self.actionAggregate_Usage.setChecked(True) else: self.actionAggregate_Usage.setChecked(False) except: self.actionAggregate_Usage.setVisible(False) self.updateDialogs = True
def _updateControls(self): self.updateDialogs=False model=self.analysisDataModel datidx= { "QDF" : self.datTypeComboBox.findText("QDF"), "ABF" : self.datTypeComboBox.findText("ABF"), "BIN" : self.datTypeComboBox.findText("BIN"), "TSV" : self.datTypeComboBox.findText("TSV") } path=model["DataFilesPath"] if len(glob.glob(format_path( str(path)+'/*qdf') )) > 0: self.datTypeComboBox.setCurrentIndex( datidx["QDF"] ) model["filter"]="*.qdf" elif len(glob.glob( format_path(str(path)+'/*abf') )) > 0: self.datTypeComboBox.setCurrentIndex( datidx["ABF"] ) model["filter"]="*.abf" elif len(glob.glob( format_path(str(path)+'/*bin') )) > 0: self.datTypeComboBox.setCurrentIndex( datidx["BIN"] ) model["filter"]="*.bin" elif len(glob.glob( format_path(str(path)+'/*dat') )) > 0: self.datTypeComboBox.setCurrentIndex( datidx["BIN"] ) model["filter"]="*.dat" elif len(glob.glob( format_path(str(path)+'/*tsv') )) > 0: self.datTypeComboBox.setCurrentIndex( datidx["TSV"] ) model["filter"]="*.tsv" elif len(glob.glob( format_path(str(path)+'/*txt') )) > 0: self.datTypeComboBox.setCurrentIndex( datidx["TSV"] ) model["filter"]="*.txt" # store the data type in the trajviewer data struct model["DataFilesType"] = str(self.datTypeComboBox.currentText()) self.startIndexLineEdit.setText(str(model["start"])) if model["end"]==-1: self.endIndexLineEdit.setText("") else: self.endIndexLineEdit.setText(str(model["end"])) self.dcOffsetDoubleSpinBox.setValue(model["dcOffset"]) if float(model["meanOpenCurr"]) == -1. or float(model["sdOpenCurr"]) == -1.: self.baselineAutoCheckBox.setChecked(True) self.OnBaselineAutoCheckbox(True) # Manually disable baseline mean and SD controls self.baselineMeanLineEdit.setText("") self.baselineSDLineEdit.setText("") self.baselineMeanLineEdit.setPlaceholderText("<auto>") self.baselineSDLineEdit.setPlaceholderText("<auto>") self.baselineMeanLineEdit.setEnabled(False) self.baselineSDLineEdit.setEnabled(False) else: # Populate baseline parameters self.baselineAutoCheckBox.setChecked(False) self.OnBaselineAutoCheckbox(False) self.baselineMeanLineEdit.setText(str(model["meanOpenCurr"])) self.baselineSDLineEdit.setText(str(model["sdOpenCurr"])) self.baselineBlockSizeDoubleSpinBox.setValue(float(model["blockSizeSec"])) # Manually enable baseline mean and SD controls self.baselineMeanLineEdit.setEnabled(True) self.baselineSDLineEdit.setEnabled(True) # Populate EventSegment parameters self._setThreshold(float(model["meanOpenCurr"]), float(model["sdOpenCurr"]), float(model["eventThreshold"])) # Populate misc parameters self.writeEventsCheckBox.setChecked(int(model["writeEventTS"])) # Populate plot widgets self.plotEventFitsCheckBox.setEnabled(int(model["writeEventTS"])) # check if parallel is available # try: # import zmq # self.parallelCheckBox.setChecked(int(model["parallelProc"])) # self.parallelCoresSpinBox.setValue(multiprocessing.cpu_count()-int(model["reserveNCPU"])) # except ImportError: # self.parallelCheckBox.hide() # self.parallelCoresSpinBox.hide() # self.parallelCoresLabel.hide() self.parallelCheckBox.hide() self.parallelCoresSpinBox.hide() self.parallelCoresLabel.hide() procidx= {} for v in self.analysisDataModel.eventProcessingAlgoKeys.keys(): procidx[v]=self.processingAlgorithmComboBox.findText(v) self.processingAlgorithmComboBox.setCurrentIndex( procidx[self.analysisDataModel.EventProcessingAlgorithmLabel()] ) # If an advanced mode dialog exists, update its settings if self.advancedSettingsDialog: if self.dataFilterDenoise: fltr=self.analysisDataModel["FilterAlgorithm"] else: fltr=None self.advancedSettingsDialog.updateSettingsString( model.GenerateSettingsView( eventPartitionAlgo=str(self.partitionAlgorithmComboBox.currentText()), eventProcessingAlgo=str(self.processingAlgorithmComboBox.currentText()), dataFilterAlgo=fltr ) ) # If the trajviewer is initialized, update the denoising settings if self.trajViewerWindow: self.trajViewerWindow.waveletLevelSpinBox.setValue(int(self.analysisDataModel["level"])) # Hide Rfb and Cfb for QDF files [control.hide() for control in [self.RfbLabel, self.qdfRfbLineEdit, self.RfbUnitsLabel, self.CfbLabel, self.qdfCfbLineEdit, self.CfbUnitsLabel]] # Set ga toggle. try: with open(resource_path("mosaic/utilities/.ga"), "r") as garead: gac = json.load(garead) if eval(gac["gauimode"]): self.actionAggregate_Usage.setVisible(True) else: self.actionAggregate_Usage.setVisible(False) if eval(gac["gaenable"]): self.actionAggregate_Usage.setChecked(True) else: self.actionAggregate_Usage.setChecked(False) except: self.actionAggregate_Usage.setVisible(False) self.updateDialogs=True
class metaTrajIO(object): """ .. warning:: |metaclass| Initialize a TrajIO object. The object can load all the data in a directory, N files from a directory or from an explicit list of filenames. In addition to the arguments defined below, implementations of this meta class may require the definition of additional arguments. See the documentation of those classes for what those may be. For example, the qdfTrajIO implementation of metaTrajIO also requires the feedback resistance (Rfb) and feedback capacitance (Cfb) to be passed at initialization. :Parameters: - `dirname` : all files from a directory ('<full path to data directory>') - `nfiles` : if requesting N files (in addition to dirname) from a specified directory - `fnames` : explicit list of filenames ([file1, file2,...]). This argument cannot be used in conjuction with dirname/nfiles. The filter argument is ignored when used in combination with fnames. - `filter` : '<wildcard filter>' (optional, filter is '*' if not specified) - `start` : Data start point in seconds. - `end` : Data end point in seconds. - `datafilter` : Handle to the algorithm to use to filter the data. If no algorithm is specified, datafilter is None and no filtering is performed. - `dcOffset` : Subtract a DC offset from the ionic current data. :Properties: - `FsHz` : sampling frequency in Hz. If the data was decimated, this property will hold the sampling frequency after decimation. - `LastFileProcessed` : return the data file that was last processed. - `ElapsedTimeSeconds` : return the analysis time in sec. :Errors: - `IncompatibleArgumentsError` : when conflicting arguments are used. - `EmptyDataPipeError` : when out of data. - `FileNotFoundError` : when data files do not exist in the specified path. - `InsufficientArgumentsError` : when incompatible arguments are passed """ __metaclass__ = ABCMeta def __init__(self, **kwargs): """ """ self.CHUNKSIZE = 10000 self.dataGenerator = None # start by setting all passed keyword arguments as class attributes for (k, v) in kwargs.iteritems(): setattr(self, k, v) # Check if the passed arguments are sane if hasattr(self, 'dirname') and hasattr(self, 'fnames'): raise IncompatibleArgumentsError( "Incompatible arguments: expect either 'dirname' or 'fnames' when initializing class {0}." .format(type(self).__name__)) # Check for the filter arg if not hasattr(self, 'filter'): self.filter = '*' if hasattr(self, 'fnames'): # set fnames here. self.dataFiles = self.fnames delattr(self, 'fnames') else: try: if hasattr(self, 'dirname') and hasattr(self, 'nfiles'): # N files from a directory self.dataFiles = self._buildFileList( self.dirname, self.filter)[:int(self.nfiles)] delattr(self, 'dirname') delattr(self, 'nfiles') elif hasattr(self, 'dirname'): # all files from a directory self.dataFiles = self._buildFileList( self.dirname, self.filter) delattr(self, 'dirname') else: raise IncompatibleArgumentsError( "Missing arguments: 'dirname' or 'fnames' must be supplied to initialize {0}" .format(type(self).__name__)) except AttributeError, err: raise IncompatibleArgumentsError(err) # set additional meta-data self.nFiles = len(self.dataFiles) self.fileFormat = 'N/A' try: sep = path_separator() self.datPath = format_path( sep.join((self.dataFiles[0].split(sep))[:-1])) except IndexError, err: raise FileNotFoundError("Files not found.")
def OnStartAnalysis(self): try: if not self.analysisRunning: if self.analysisDataModel["DataFilesPath"]: self._setEnableSettingsWidgets(False) self._setEnableDataSettingsWidgets(False) if self.dataFilterDenoise: fltr=self.analysisDataModel["FilterAlgorithm"] else: fltr=None with open(format_path(self.analysisDataModel["DataFilesPath"]+"/.settings"), 'w') as f: f.write( self.analysisDataModel.GenerateSettingsView( eventPartitionAlgo=str(self.partitionAlgorithmComboBox.currentText()), eventProcessingAlgo=str(self.processingAlgorithmComboBox.currentText()), dataFilterAlgo=fltr ) ) self.startAnalysisPushButton.setEnabled(False) self.actionStart_Analysis.setEnabled(False) self.startAnalysisPushButton.setStyleSheet("") self.startAnalysisPushButton.setText("Starting...") self.actionStart_Analysis.setText("Starting...") self.trajViewerWindow.hide() # Query the number of database files in the analysis directory self.nDBFiles=len(self._getdbfiles()) # setup the worker thread self.aThread=QtCore.QThread(parent=self) self.aWorker=analysisworker.analysisWorker( self.analysisDataModel.GenerateAnalysisObject( eventPartitionAlgo=str(self.partitionAlgorithmComboBox.currentText()), eventProcessingAlgo=str(self.processingAlgorithmComboBox.currentText()), dataFilterAlgo=fltr ) ) self.aWorker.moveToThread(self.aThread) self.aWorker.analysisFinished.connect(self.OnAnalysisFinished) # Handle the threads finished signal # self.aThread.finished.connect(self.OnThreadFinished) # self.aThread.finished.connect(self.aThread.quit) self.aThread.start() # Start the analysis QtCore.QMetaObject.invokeMethod(self.aWorker, 'startAnalysis', Qt.QueuedConnection) self.startButtonClicked=True else: # Stop the analysis QtCore.QMetaObject.invokeMethod(self.aWorker, 'stopAnalysis', Qt.QueuedConnection) self.startAnalysisPushButton.setEnabled(False) self.actionStart_Analysis.setEnabled(False) # self._setEnableSettingsWidgets(True) # self._setEnableDataSettingsWidgets(True) self.startAnalysisPushButton.setStyleSheet("") self.startAnalysisPushButton.setText("Stopping...") self.actionStart_Analysis.setText("Stopping...") # self.analysisRunning=False except FileNotFoundError: QtGui.QMessageBox.warning(self, "Data Error", "No data files found in " + str(self.analysisDataModel["DataFilesPath"]) ) self._setEnableSettingsWidgets(True) self._setEnableDataSettingsWidgets(True) self.startAnalysisPushButton.setStyleSheet("") self.startAnalysisPushButton.setText("Start Analysis") self.actionStart_Analysis.setText("Start Analysis")
log.info("Logs will be saved to: {0}".format(logdir)) except AttributeError, err: if sys.platform.startswith('darwin'): logdir=format_path(os.path.expanduser('~')+"/Library/Logs/MOSAIC") if not os.path.exists(logdir): os.mkdir(logdir) elif sys.platform.startswith('linux'): if os.getuid()==0: logdir="/var/log/" else: log.info("MOSAIC log will be saved to ~/mosaic.log. Run MOSAIC with sudo to save logs to '/var/log/.") logdir=os.path.expanduser("~") else: logdir=os.path.expanduser("~") logname=format_path(logdir+"/mosaic.log") rfh=logging.handlers.RotatingFileHandler(filename=logname, maxBytes=mosaic.LogSizeBytes, backupCount=5) rfh.setFormatter(formatstr) if mosaic.DeveloperMode: rfh.setLevel(logging.DEBUG) else: rfh.setLevel(logging.INFO) log.addHandler(rfh) sh=None def __init__(self, *args, **kwargs): pass