def __init__(self, host,port, parent,runOnlyOnce=False): self.host =host self.port = port self.inqueue = deque([],10000) self.parent = parent self.datareceiver = DataReceiver(self.host,self.port, self,None) if runOnlyOnce: self.stopAfterNextConnection() self.datareceiver.start() self.hasShutdown=False
def handleOfferMessage(incomeMessage, ip_addr): # check the resource table, if not found, send a ack, else, send a nack for record in self.resourceTable.records: if record.fid == incomeMessage.records[0] and record.status != RecordStatus.on_remote: # send a nack nack = NACK() nack.addRecord(incomeMessage.records[0]) nack.send(ip_addr, self.port, self.interface) break # update the resource table record = Record(incomeMessage.records[0], '127.0.0.1', RecordStatus.on_the_wire) self.resourceTable.records.append(record) rc = -1 while rc != 0: # setup a data receiver receiver = DataReceiver() # send an ack ack = ACK(receiver.port) ack.addRecord(incomeMessage.records[0]) ack.send(ip_addr, self.port, self.interface) # start the receiver rc = receiver.receive(self.data_directory + "/" + incomeMessage.records[0].decode(encoding='UTF-8')) # update resource table self.resourceTable.updateStatus(incomeMessage.records[0], '127.0.0.1', RecordStatus.on_the_disk)
class Consumer(): class ConsumerException(Exception): def __init__(self, msg): self.msg = msg def __init__(self, host,port, parent,runOnlyOnce=False): self.host =host self.port = port self.inqueue = deque([],10000) self.parent = parent self.datareceiver = DataReceiver(self.host,self.port, self,None) if runOnlyOnce: self.stopAfterNextConnection() self.datareceiver.start() self.hasShutdown=False # def __del__(self): # print 'consumer ', self.host, 'died' def shutdownNow(self): self.hasShutdown=True self.datareceiver.stopReceiving() self.parent = None self.inqueue=None def stopAfterNextConnection(self): self.datareceiver.shutdownAfterConnection(True) def waitData(self,timeout=None): while len(self.inqueue)==0: self.datareceiver.waitData(timeout) if self.hasShutdown: return if len(self.inqueue)==0 and self.datareceiver.receiving==False: # only raise an exception once all data has been eaten up raise Consumer.ConsumerException("Receiver stopped") def getData(self): if(len(self.inqueue) > 0): return self.inqueue.popleft() else: return None def getLatest(self): if(len(self.inqueue) > 0): return self.inqueue[-1] else: return None def setData(self,data,callbackData): if not self.hasShutdown: self.inqueue.append(data) if len(self.inqueue)>9990: #print "Consumer overflow",self.host,self.port pass def getConnectionCount(self): return self.datareceiver.getConnectionCount() def backchannelSendData(self,data): self.datareceiver.backchannelSendData(data)
def createStream(self,streamID,friendlyName,hostAddress,hostAlias,query,port,dataTable,live,startTime): dataTable=self.removeNonAlphabetCharacters(str(dataTable)) # search for streams table for existing stream ID, or create that stream ID etc in table self.createStreamsTableIfNeeded() currentID=-1 dbCur = self.dbConnection.cursor() if streamID==-1: if hostAlias=="" or port==0 or dataTable=="": print "Help: trying to create stream without the right parameters" sys.exit(-1) # find a stream with matching host alias, port and data table - friendly name is just for information, and host address is just the current connection address dbCur.execute("select _id from streams where hostAlias=? and port = ? and query= ? and dataTable=?",(hostAlias,port,query,dataTable)) currentID=dbCur.fetchone() if currentID == None: if live==-1: # default live to 1 for new streams live = 1 # this with means that it automatically commits the changes on success with self.dbConnection: self.dbConnection.execute("insert into streams(hostAddress,hostAlias,port,query,friendlyName,dataTable,live) values(?,?,?,?,?,?,1)",(hostAddress,hostAlias,port,query,friendlyName,dataTable)) dbCur.execute("select _id from streams where hostAlias=? and port = ? and query=? and dataTable=?",(hostAlias,port,query,dataTable)) currentID= dbCur.fetchone() if currentID == None: print "Help, couldn't insert new stream into streams table" sys.exit(-1) currentID =int(currentID[0]) else: # check that this stream exists #print streamID dbCur.execute("select _id,hostAddress,dataTable,live,query,hostAlias,port,friendlyName from streams where _id=%d"%(streamID)) dbRow=dbCur.fetchone() if dbRow == None: print "Help: trying to alter a stream (id=%d) which doesn't exist"%currentID sys.exit(-1) currentID=dbRow[0] # if we haven't posted a host address, then add this one if len(hostAddress)==0: hostAddress=dbRow[1] #data table forces to the same table dataTable=dbRow[2] # liveness is read from the table if it isn't set explicitly if live==-1: live=int(dbRow[3]) #read in the query always - don't allow query changes without making a new stream query=dbRow[4] #read in the host alias always - don't allow query changes without making a new stream hostAlias=dbRow[5] #read in the port always - don't allow port changes without making a new stream port=int(dbRow[6]) if len(friendlyName)==0: friendlyName=dbRow[7] #print "ID:",currentID # update the stream host address, friendly name and liveness in the streams table in case they have changed and commit with self.dbConnection: #print (friendlyName,hostAddress,hostAlias,port,query,dataTable,live) if len(friendlyName)>0: self.dbConnection.execute("update streams set friendlyName=? where _id=?",(friendlyName,currentID)) if len(hostAddress)>0: self.dbConnection.execute("update streams set hostAddress=? where _id=?",(hostAddress,currentID)) if live!=-1: self.dbConnection.execute("update streams set live=%d where _id=%d"%(live,currentID)) # self.dbConnection.execute("update streams set friendlyName=?,hostAddress=?,live=? where _id=?",(friendlyName,hostAddress,live,currentID)) self.createdataTableIfNeeded(dataTable) # open connection reader to ip address, port if len(query)>0: hostAddress=query+'@'+hostAddress if live==1: # see if a receiver already exists alreadyReading=False for c in range(0,len(self.inStreams)): (rec,fri,tab,id,oldTime)=self.inStreams[c] if id==currentID: alreadyReading=True #set a new start time self.inStreams[c]=(rec,fri,tab,id,startTime) if not alreadyReading: receiver=DataReceiver(hostAddress,port, self,(dataTable,currentID,startTime)) self.inStreams.append( (receiver,friendlyName,dataTable,currentID,startTime) ) receiver.start() print "Opened reader for %s at %s[%s]:%d"%(friendlyName,hostAlias,hostAddress,port) else: print "Already reading %s at %s[%s]:%d"%(friendlyName,hostAlias,hostAddress,port) else: foundPos=-1 foundReceiver=None for c in range(0,len(self.inStreams)): (rec,fri,tab,id,oldStartTime) =self.inStreams[c] if id==currentID: foundPos=c foundReceiver=rec if foundPos!=-1: self.inStreams.pop(foundPos) foundReceiver.stopReceiving()