if importStreamDetails.has_key(signalNum): (signal,streamID,preOffset,scaling,postOffset)=importStreamDetails[signalNum] if dbConnection.write16BitImportData(streamID,dataBuf,timeStart,timePerSample,preOffset=preOffset,scaling=scaling,postOffset=postOffset)==False: print "Error writing import data - stopping" dbConnection.stopSessionImporting(importSession) return False totalSamples+=numSamples curRecords+=1 timeStart+=timePerRecord print "Written: %d records = %d data points"%(curRecords,totalSamples) # stop the session importing (doesn't really do very much, except let # the db connection know it can drop any import connections it has held open dbConnection.stopSessionImporting(importSession) return True if __name__ == '__main__': db=DataStoreConnection("127.0.0.1",49990) while db.checkConnection()==False: print "Can't connect to database" time.sleep(1.0) r=EDFReader("test.eda",EDFReader.EDA_FILE) print r.getEDFHeaderInfo() print r.getEDFStreamInfo(0) print r.getEDFStreamInfo(1) print r.getEDFStreamInfo(2) print r.getEDFStreamInfo(3) if not r.readEDFSignalIntoDBConnection(db,[5],"Woo"): print "Failed to read edf into DB" # if not r.readEDFSignalIntoDBConnection(db,[0,1,2,3,4,5],"Woo"): # print "Failed to read edf into DB"
def run(self): self.dbMode=False if len(sys.argv)>1 and sys.argv[1].lower()=="-xmlinfo": self.showXMLInfo() return if len(sys.argv)>1 and sys.argv[1].lower()=="-dbprocess": if not self.supportsDBMode: self.showUsage() return self.dbMode=True self.numExpectedArgsMin=5+self.numInputs + self.numOutputs +self.numArguments # parse arguments if len(sys.argv)<self.numExpectedArgsMin+1: self.showUsage() return try: if self.dbMode: # read from database and process it into another db stream curArg=2 # get DB details self.dbHost=sys.argv[curArg] curArg+=1 self.dbPort=int(sys.argv[curArg]) curArg+=1 self.dbStartTime=float(sys.argv[curArg]) curArg+=1 self.dbDuration=float(sys.argv[curArg]) curArg+=1 self.dbEndTime=self.dbStartTime+self.dbDuration print self.dbHost,self.dbPort self.dbConnection=DataStoreConnection(self.dbHost,int(self.dbPort)) if not self.dbConnection.checkConnection(): print "Couldn't connect to database" return -1 # make receivers self.dataReceivers=[] self.inQueues=[] self.waitingData=[] self.lastValues=[] self.dbStreamIDs=[] for c in range(0,self.numInputs): if curArg>=len(sys.argv): self.showUsage() return if self.inputs[c][0].startswith('*'): while (not sys.argv[curArg].startswith('*')) and curArg<len(sys.argv): # variable number of input sources, with a '*' at the end inStream=int(sys.argv[curArg]) self.dbStreamIDs.append(inStream) curArg+=1 newReceiver=DataReceiver("<cursor id='%d'/>@%s"%(inStream,self.dbHost),self.dbPort,self,len(self.dataReceivers)) self.dataReceivers.append(newReceiver) self.inQueues.append(deque(maxlen=1000)) self.waitingData.append(False) self.lastValues.append("0.0") curArg+=1 else: inStream=int(sys.argv[curArg]) self.dbStreamIDs.append(inStream) curArg+=1 newReceiver=DataReceiver("<cursor id='%d'/>@%s"%(inStream,self.dbHost),self.dbPort,self,len(self.dataReceivers)) self.dataReceivers.append(newReceiver) self.inQueues.append(deque(maxlen=1000)) self.lastValues.append("0.0") self.waitingData.append(False) self.dbExistingSession=None self.dbSessionName="Processed data" for id in self.dbStreamIDs: if self.dbExistingSession==None: sessions=self.dbConnection.listSessions(int(id)) print sessions if sessions!=None and len(sessions)>0: self.dbExistingSession=int(sessions[0][0]) self.dbSessionName=None print "Existing session:%d (%s)"%(self.dbExistingSession,sessions[0][1]) if self.dbStartTime==-1 or self.dbDuration==-1: streamTimes=self.dbConnection.getStreamTimes([int(id)]) print streamTimes if streamTimes!=None: if self.dbStartTime==-1: self.dbStartTime=streamTimes[0][1] if self.dbDuration==-1: self.dbEndTime=streamTimes[0][2] self.dbDuration=streamTimes[0][2]-streamTimes[0][1] self.dbEndTime=self.dbStartTime+self.dbDuration # read the output stream names self.outputNames={} self.outputBuffers={} self.outputStreams={} self.createStreamNames=[] for c in range(0,self.numOutputs): if sys.argv[curArg].lower()!="none": self.outputNames[c]=sys.argv[curArg] self.outputBuffers[c]=deque(maxlen=512) self.createStreamNames.append(sys.argv[curArg]) curArg+=1 self.outputStreamsSorted=sorted(self.outputNames.items()) self.outputSession=0 self.outputStreamList=[] if len(self.outputNames)>0: self.outputSession,self.outputStreamList=self.dbConnection.startSessionImporting(self.dbStartTime,self.createStreamNames,sessionName=self.dbSessionName,existingSessionID=self.dbExistingSession) for ((outCount,name),strmID) in zip(self.outputStreamsSorted,self.outputStreamList): self.outputStreams[outCount]=int(strmID) # read the argument values self.argumentValues=[] for c in range(0,self.numArguments): if curArg>=len(sys.argv): self.showUsage() return self.argumentValues.append(sys.argv[curArg]) # if it is a float, make the argument value be a float try: self.argumentValues[-1]=float(self.argumentValues[-1]) except ValueError: None curArg+=1 # pretend that it is the db start time self.processArguments(self.dbStartTime) # now start receivers and senders for (c,recv) in enumerate(self.dataReceivers): recv.start() print "Connecting to database" doneConnection=False connectionAttempts=0 while not doneConnection: time.sleep(0.5) doneConnection=True for (c,recv) in enumerate(self.dataReceivers): if recv.isConnected()==False: doneConnection=False connectionAttempts+=1 if connectionAttempts==5: print "Error - couldn't connect receiver to database" return -1 for (c,recv) in enumerate(self.dataReceivers): self.waitingData[c]=True # fill initial buffers recv.backchannelSendData("OFFSET %f 512"%(self.dbStartTime)) self.dbTime=self.dbStartTime numPoints=0 numWrites=0 startProcessingTime=time.time() percentTime=(self.dbEndTime-self.dbStartTime)/100.0 nextPercentTime=self.dbStartTime+percentTime percentage=0 while self.dbTime<self.dbEndTime: while self.dbTime>nextPercentTime: percentage+=1 nextPercentTime+=percentTime print "DBProcessing: %d%% complete,%d points read,%d points written"%(percentage,numPoints,numWrites) sys.stdout.flush() minTime=self.dbEndTime minQueue=None emptyQueue=False minQueueNum=None # find the earliest datapoint for c,queue in enumerate(self.inQueues): if len(queue)==0 and self.waitingData[c]==True: # if one of the queues is empty and waiting for more data, we have to wait for it to be filled # before we can work out what is the shortest point print "Waiting data" time.sleep(0.1) emptyQueue=True minQueue=None break elif len(queue)>0 and queue[0][0]<minTime: minTime=min(minTime,queue[0][0]) minQueue=queue minQueueNum=c if minQueue!=None and minTime<self.dbEndTime: # process this point timestamp,value=minQueue.popleft() self.lastValues[minQueueNum]=value self.dbTime=timestamp self.process(timestamp,self.lastValues,minQueueNum) numPoints+=1 # print "Process: ",timestamp,self.lastValues # get more data if needed for c,queue in enumerate(self.inQueues): if len(queue)<256 and len(queue)>0 and self.waitingData[c]==False: lastTime=queue[-1][0] self.waitingData[c]=True self.dataReceivers[c].backchannelSendData("OFFSET %f %d"%(lastTime,999-len(queue))) # write output buffers if they are big enough for streamNum,buffer in self.outputBuffers.items(): if ((not emptyQueue) and minTime>=self.dbEndTime and len(buffer)>0) or len(buffer)>=256: numWrites+=len(buffer) self.dbConnection.writeTimestampedImportData(self.outputStreams[streamNum],buffer) buffer.clear() if (not emptyQueue) and minTime>=self.dbEndTime: # no data left break print "DBProcessing: 100%% complete,%d points read,%d points written"%(numPoints,numWrites) print "Done processing, %d data points read, %d data points written, %.2f seconds"%(numPoints,numWrites,time.time()-startProcessingTime) sys.stdout.flush() for c in self.dataReceivers: c.stopReceiving() # receivers will only stop once some data comes to them, so force it with a backchannel send c.backchannelSendData("OFFSET 0 0") print "Ended receivers" return # we're saving alright, so now step through all the db streams point by point, getting values greater than the current value # we have to do cunning things in receive callbacks to fill queues # # first: seek all dbcursors to the start time and return a buffer full forwards (which goes into a queue for that stream) # take the earliest point off any of the receive stream queues # update last values and call process with that timestamp # # in db mode, addProcessedValues should add the current timestamp # # if a stream buffer is > half empty, send a request to refill # if a stream buffer is empty, wait till request ends on that stream # if the request ends and the buffer is empty, mark that stream as done (replace it with none) # else: # normal live mode curArg=1 # make receivers self.dataReceivers=[] self.lastValues=[] for c in range(0,self.numInputs): if curArg>=len(sys.argv): self.showUsage() return if self.inputs[c][0].startswith('*'): while (not sys.argv[curArg].startswith('*')) and curArg<len(sys.argv): # variable number of input sources, with a '*' at the end inHost=sys.argv[curArg] curArg+=1 inPort=int(sys.argv[curArg]) curArg+=1 newReceiver=DataReceiver(inHost,inPort,self,len(self.dataReceivers)) self.dataReceivers.append(newReceiver) self.lastValues.append("0.0") curArg+=1 else: inHost=sys.argv[curArg] curArg+=1 inPort=int(sys.argv[curArg]) curArg+=1 newReceiver=DataReceiver(inHost,inPort,self,len(self.dataReceivers)) self.dataReceivers.append(newReceiver) self.lastValues.append("0.0") outPort=int(sys.argv[curArg]) curArg+=1 # make sender(s) and queues self.dataSenders=[] if self.numOutputs==1: self.dataSenders.append(DataSender(outPort, self)) else: self.senderParent=SenderParent(outPort,self) for c in range(0,self.numOutputs): self.dataSenders.append (ChildDataSender(c,self)) self.argumentValues=[] for c in range(0,self.numArguments): if curArg>=len(sys.argv): self.showUsage() return self.argumentValues.append(sys.argv[curArg]) # if it is a float, make the argument value be a float try: self.argumentValues[-1]=float(self.argumentValues[-1]) except ValueError: None curArg+=1 if self.canHaveGui: if curArg<len(sys.argv): if sys.argv[curArg]=="1" or sys.argv[curArg].lower()=="true" or sys.argv[curArg].lower()=="yes": self.hasGUI=True self.guiTitle="%s [%d]"%(self.processorName,outPort) curArg+=1 if curArg<len(sys.argv): self.guiTitle=sys.argv[curArg] curArg+=1 self._initialClock=time.clock() self._initialTime=time.time() self.processArguments(self._initialTime) # now start receivers and senders for c in self.dataReceivers: c.start() for c in self.dataSenders: c.setPush(True) c.start() if self.numOutputs>1: self.senderParent.start() # now show gui if needed (this blocks until the gui is closed) if self.hasGUI: self.runGUI(self.guiSetup,self.guiTitle) else: # now just hold on - processing is done in response to the receivers calling setData # and output is done when the process function calls addProcessedValues while True: time.sleep(1000.0) except IOError,e: print e self.showUsage() return