if importStreamDetails.has_key(signalNum):
(signal,streamID,preOffset,scaling,postOffset)=importStreamDetails[signalNum]
if dbConnection.write16BitImportData(streamID,dataBuf,timeStart,timePerSample,preOffset=preOffset,scaling=scaling,postOffset=postOffset)==False:
print "Error writing import data - stopping"
dbConnection.stopSessionImporting(importSession)
return False
totalSamples+=numSamples
curRecords+=1
timeStart+=timePerRecord
print "Written: %d records = %d data points"%(curRecords,totalSamples)
# stop the session importing (doesn't really do very much, except let
# the db connection know it can drop any import connections it has held open
dbConnection.stopSessionImporting(importSession)
return True
if __name__ == '__main__':
db=DataStoreConnection("127.0.0.1",49990)
while db.checkConnection()==False:
print "Can't connect to database"
time.sleep(1.0)
r=EDFReader("test.eda",EDFReader.EDA_FILE)
print r.getEDFHeaderInfo()
print r.getEDFStreamInfo(0)
print r.getEDFStreamInfo(1)
print r.getEDFStreamInfo(2)
print r.getEDFStreamInfo(3)
if not r.readEDFSignalIntoDBConnection(db,[5],"Woo"):
print "Failed to read edf into DB"
# if not r.readEDFSignalIntoDBConnection(db,[0,1,2,3,4,5],"Woo"):
# print "Failed to read edf into DB"
def run(self):
self.dbMode=False
if len(sys.argv)>1 and sys.argv[1].lower()=="-xmlinfo":
self.showXMLInfo()
return
if len(sys.argv)>1 and sys.argv[1].lower()=="-dbprocess":
if not self.supportsDBMode:
self.showUsage()
return
self.dbMode=True
self.numExpectedArgsMin=5+self.numInputs + self.numOutputs +self.numArguments
# parse arguments
if len(sys.argv)<self.numExpectedArgsMin+1:
self.showUsage()
return
try:
if self.dbMode:
# read from database and process it into another db stream
curArg=2
# get DB details
self.dbHost=sys.argv[curArg]
curArg+=1
self.dbPort=int(sys.argv[curArg])
curArg+=1
self.dbStartTime=float(sys.argv[curArg])
curArg+=1
self.dbDuration=float(sys.argv[curArg])
curArg+=1
self.dbEndTime=self.dbStartTime+self.dbDuration
print self.dbHost,self.dbPort
self.dbConnection=DataStoreConnection(self.dbHost,int(self.dbPort))
if not self.dbConnection.checkConnection():
print "Couldn't connect to database"
return -1
# make receivers
self.dataReceivers=[]
self.inQueues=[]
self.waitingData=[]
self.lastValues=[]
self.dbStreamIDs=[]
for c in range(0,self.numInputs):
if curArg>=len(sys.argv):
self.showUsage()
return
if self.inputs[c][0].startswith('*'):
while (not sys.argv[curArg].startswith('*')) and curArg<len(sys.argv):
# variable number of input sources, with a '*' at the end
inStream=int(sys.argv[curArg])
self.dbStreamIDs.append(inStream)
curArg+=1
newReceiver=DataReceiver("<cursor id='%d'/>@%s"%(inStream,self.dbHost),self.dbPort,self,len(self.dataReceivers))
self.dataReceivers.append(newReceiver)
self.inQueues.append(deque(maxlen=1000))
self.waitingData.append(False)
self.lastValues.append("0.0")
curArg+=1
else:
inStream=int(sys.argv[curArg])
self.dbStreamIDs.append(inStream)
curArg+=1
newReceiver=DataReceiver("<cursor id='%d'/>@%s"%(inStream,self.dbHost),self.dbPort,self,len(self.dataReceivers))
self.dataReceivers.append(newReceiver)
self.inQueues.append(deque(maxlen=1000))
self.lastValues.append("0.0")
self.waitingData.append(False)
self.dbExistingSession=None
self.dbSessionName="Processed data"
for id in self.dbStreamIDs:
if self.dbExistingSession==None:
sessions=self.dbConnection.listSessions(int(id))
print sessions
if sessions!=None and len(sessions)>0:
self.dbExistingSession=int(sessions[0][0])
self.dbSessionName=None
print "Existing session:%d (%s)"%(self.dbExistingSession,sessions[0][1])
if self.dbStartTime==-1 or self.dbDuration==-1:
streamTimes=self.dbConnection.getStreamTimes([int(id)])
print streamTimes
if streamTimes!=None:
if self.dbStartTime==-1:
self.dbStartTime=streamTimes[0][1]
if self.dbDuration==-1:
self.dbEndTime=streamTimes[0][2]
self.dbDuration=streamTimes[0][2]-streamTimes[0][1]
self.dbEndTime=self.dbStartTime+self.dbDuration
# read the output stream names
self.outputNames={}
self.outputBuffers={}
self.outputStreams={}
self.createStreamNames=[]
for c in range(0,self.numOutputs):
if sys.argv[curArg].lower()!="none":
self.outputNames[c]=sys.argv[curArg]
self.outputBuffers[c]=deque(maxlen=512)
self.createStreamNames.append(sys.argv[curArg])
curArg+=1
self.outputStreamsSorted=sorted(self.outputNames.items())
self.outputSession=0
self.outputStreamList=[]
if len(self.outputNames)>0:
self.outputSession,self.outputStreamList=self.dbConnection.startSessionImporting(self.dbStartTime,self.createStreamNames,sessionName=self.dbSessionName,existingSessionID=self.dbExistingSession)
for ((outCount,name),strmID) in zip(self.outputStreamsSorted,self.outputStreamList):
self.outputStreams[outCount]=int(strmID)
# read the argument values
self.argumentValues=[]
for c in range(0,self.numArguments):
if curArg>=len(sys.argv):
self.showUsage()
return
self.argumentValues.append(sys.argv[curArg])
# if it is a float, make the argument value be a float
try:
self.argumentValues[-1]=float(self.argumentValues[-1])
except ValueError:
None
curArg+=1
# pretend that it is the db start time
self.processArguments(self.dbStartTime)
# now start receivers and senders
for (c,recv) in enumerate(self.dataReceivers):
recv.start()
print "Connecting to database"
doneConnection=False
connectionAttempts=0
while not doneConnection:
time.sleep(0.5)
doneConnection=True
for (c,recv) in enumerate(self.dataReceivers):
if recv.isConnected()==False:
doneConnection=False
connectionAttempts+=1
if connectionAttempts==5:
print "Error - couldn't connect receiver to database"
return -1
for (c,recv) in enumerate(self.dataReceivers):
self.waitingData[c]=True
# fill initial buffers
recv.backchannelSendData("OFFSET %f 512"%(self.dbStartTime))
self.dbTime=self.dbStartTime
numPoints=0
numWrites=0
startProcessingTime=time.time()
percentTime=(self.dbEndTime-self.dbStartTime)/100.0
nextPercentTime=self.dbStartTime+percentTime
percentage=0
while self.dbTime<self.dbEndTime:
while self.dbTime>nextPercentTime:
percentage+=1
nextPercentTime+=percentTime
print "DBProcessing: %d%% complete,%d points read,%d points written"%(percentage,numPoints,numWrites)
sys.stdout.flush()
minTime=self.dbEndTime
minQueue=None
emptyQueue=False
minQueueNum=None
# find the earliest datapoint
for c,queue in enumerate(self.inQueues):
if len(queue)==0 and self.waitingData[c]==True:
# if one of the queues is empty and waiting for more data, we have to wait for it to be filled
# before we can work out what is the shortest point
print "Waiting data"
time.sleep(0.1)
emptyQueue=True
minQueue=None
break
elif len(queue)>0 and queue[0][0]<minTime:
minTime=min(minTime,queue[0][0])
minQueue=queue
minQueueNum=c
if minQueue!=None and minTime<self.dbEndTime:
# process this point
timestamp,value=minQueue.popleft()
self.lastValues[minQueueNum]=value
self.dbTime=timestamp
self.process(timestamp,self.lastValues,minQueueNum)
numPoints+=1
# print "Process: ",timestamp,self.lastValues
# get more data if needed
for c,queue in enumerate(self.inQueues):
if len(queue)<256 and len(queue)>0 and self.waitingData[c]==False:
lastTime=queue[-1][0]
self.waitingData[c]=True
self.dataReceivers[c].backchannelSendData("OFFSET %f %d"%(lastTime,999-len(queue)))
# write output buffers if they are big enough
for streamNum,buffer in self.outputBuffers.items():
if ((not emptyQueue) and minTime>=self.dbEndTime and len(buffer)>0) or len(buffer)>=256:
numWrites+=len(buffer)
self.dbConnection.writeTimestampedImportData(self.outputStreams[streamNum],buffer)
buffer.clear()
if (not emptyQueue) and minTime>=self.dbEndTime:
# no data left
break
print "DBProcessing: 100%% complete,%d points read,%d points written"%(numPoints,numWrites)
print "Done processing, %d data points read, %d data points written, %.2f seconds"%(numPoints,numWrites,time.time()-startProcessingTime)
sys.stdout.flush()
for c in self.dataReceivers:
c.stopReceiving()
# receivers will only stop once some data comes to them, so force it with a backchannel send
c.backchannelSendData("OFFSET 0 0")
print "Ended receivers"
return
# we're saving alright, so now step through all the db streams point by point, getting values greater than the current value
# we have to do cunning things in receive callbacks to fill queues
#
# first: seek all dbcursors to the start time and return a buffer full forwards (which goes into a queue for that stream)
# take the earliest point off any of the receive stream queues
# update last values and call process with that timestamp
#
# in db mode, addProcessedValues should add the current timestamp
#
# if a stream buffer is > half empty, send a request to refill
# if a stream buffer is empty, wait till request ends on that stream
# if the request ends and the buffer is empty, mark that stream as done (replace it with none)
#
else:
# normal live mode
curArg=1
# make receivers
self.dataReceivers=[]
self.lastValues=[]
for c in range(0,self.numInputs):
if curArg>=len(sys.argv):
self.showUsage()
return
if self.inputs[c][0].startswith('*'):
while (not sys.argv[curArg].startswith('*')) and curArg<len(sys.argv):
# variable number of input sources, with a '*' at the end
inHost=sys.argv[curArg]
curArg+=1
inPort=int(sys.argv[curArg])
curArg+=1
newReceiver=DataReceiver(inHost,inPort,self,len(self.dataReceivers))
self.dataReceivers.append(newReceiver)
self.lastValues.append("0.0")
curArg+=1
else:
inHost=sys.argv[curArg]
curArg+=1
inPort=int(sys.argv[curArg])
curArg+=1
newReceiver=DataReceiver(inHost,inPort,self,len(self.dataReceivers))
self.dataReceivers.append(newReceiver)
self.lastValues.append("0.0")
outPort=int(sys.argv[curArg])
curArg+=1
# make sender(s) and queues
self.dataSenders=[]
if self.numOutputs==1:
self.dataSenders.append(DataSender(outPort, self))
else:
self.senderParent=SenderParent(outPort,self)
for c in range(0,self.numOutputs):
self.dataSenders.append (ChildDataSender(c,self))
self.argumentValues=[]
for c in range(0,self.numArguments):
if curArg>=len(sys.argv):
self.showUsage()
return
self.argumentValues.append(sys.argv[curArg])
# if it is a float, make the argument value be a float
try:
self.argumentValues[-1]=float(self.argumentValues[-1])
except ValueError:
None
curArg+=1
if self.canHaveGui:
if curArg<len(sys.argv):
if sys.argv[curArg]=="1" or sys.argv[curArg].lower()=="true" or sys.argv[curArg].lower()=="yes":
self.hasGUI=True
self.guiTitle="%s [%d]"%(self.processorName,outPort)
curArg+=1
if curArg<len(sys.argv):
self.guiTitle=sys.argv[curArg]
curArg+=1
self._initialClock=time.clock()
self._initialTime=time.time()
self.processArguments(self._initialTime)
# now start receivers and senders
for c in self.dataReceivers:
c.start()
for c in self.dataSenders:
c.setPush(True)
c.start()
if self.numOutputs>1:
self.senderParent.start()
# now show gui if needed (this blocks until the gui is closed)
if self.hasGUI:
self.runGUI(self.guiSetup,self.guiTitle)
else:
# now just hold on - processing is done in response to the receivers calling setData
# and output is done when the process function calls addProcessedValues
while True:
time.sleep(1000.0)
except IOError,e:
print e
self.showUsage()
return
