def __init__(self, host,port, parent,runOnlyOnce=False):
self.host =host
self.port = port
self.inqueue = deque([],10000)
self.parent = parent
self.datareceiver = DataReceiver(self.host,self.port, self,None)
if runOnlyOnce:
self.stopAfterNextConnection()
self.datareceiver.start()
self.hasShutdown=False
def handleOfferMessage(incomeMessage, ip_addr):
# check the resource table, if not found, send a ack, else, send a nack
for record in self.resourceTable.records:
if record.fid == incomeMessage.records[0] and record.status != RecordStatus.on_remote:
# send a nack
nack = NACK()
nack.addRecord(incomeMessage.records[0])
nack.send(ip_addr, self.port, self.interface)
break
# update the resource table
record = Record(incomeMessage.records[0], '127.0.0.1', RecordStatus.on_the_wire)
self.resourceTable.records.append(record)
rc = -1
while rc != 0:
# setup a data receiver
receiver = DataReceiver()
# send an ack
ack = ACK(receiver.port)
ack.addRecord(incomeMessage.records[0])
ack.send(ip_addr, self.port, self.interface)
# start the receiver
rc = receiver.receive(self.data_directory + "/" + incomeMessage.records[0].decode(encoding='UTF-8'))
# update resource table
self.resourceTable.updateStatus(incomeMessage.records[0], '127.0.0.1', RecordStatus.on_the_disk)
class Consumer():
class ConsumerException(Exception):
def __init__(self, msg):
self.msg = msg
def __init__(self, host,port, parent,runOnlyOnce=False):
self.host =host
self.port = port
self.inqueue = deque([],10000)
self.parent = parent
self.datareceiver = DataReceiver(self.host,self.port, self,None)
if runOnlyOnce:
self.stopAfterNextConnection()
self.datareceiver.start()
self.hasShutdown=False
# def __del__(self):
# print 'consumer ', self.host, 'died'
def shutdownNow(self):
self.hasShutdown=True
self.datareceiver.stopReceiving()
self.parent = None
self.inqueue=None
def stopAfterNextConnection(self):
self.datareceiver.shutdownAfterConnection(True)
def waitData(self,timeout=None):
while len(self.inqueue)==0:
self.datareceiver.waitData(timeout)
if self.hasShutdown:
return
if len(self.inqueue)==0 and self.datareceiver.receiving==False:
# only raise an exception once all data has been eaten up
raise Consumer.ConsumerException("Receiver stopped")
def getData(self):
if(len(self.inqueue) > 0):
return self.inqueue.popleft()
else:
return None
def getLatest(self):
if(len(self.inqueue) > 0):
return self.inqueue[-1]
else:
return None
def setData(self,data,callbackData):
if not self.hasShutdown:
self.inqueue.append(data)
if len(self.inqueue)>9990:
#print "Consumer overflow",self.host,self.port
pass
def getConnectionCount(self):
return self.datareceiver.getConnectionCount()
def backchannelSendData(self,data):
self.datareceiver.backchannelSendData(data)
def createStream(self,streamID,friendlyName,hostAddress,hostAlias,query,port,dataTable,live,startTime):
dataTable=self.removeNonAlphabetCharacters(str(dataTable))
# search for streams table for existing stream ID, or create that stream ID etc in table
self.createStreamsTableIfNeeded()
currentID=-1
dbCur = self.dbConnection.cursor()
if streamID==-1:
if hostAlias=="" or port==0 or dataTable=="":
print "Help: trying to create stream without the right parameters"
sys.exit(-1)
# find a stream with matching host alias, port and data table - friendly name is just for information, and host address is just the current connection address
dbCur.execute("select _id from streams where hostAlias=? and port = ? and query= ? and dataTable=?",(hostAlias,port,query,dataTable))
currentID=dbCur.fetchone()
if currentID == None:
if live==-1:
# default live to 1 for new streams
live = 1
# this with means that it automatically commits the changes on success
with self.dbConnection:
self.dbConnection.execute("insert into streams(hostAddress,hostAlias,port,query,friendlyName,dataTable,live) values(?,?,?,?,?,?,1)",(hostAddress,hostAlias,port,query,friendlyName,dataTable))
dbCur.execute("select _id from streams where hostAlias=? and port = ? and query=? and dataTable=?",(hostAlias,port,query,dataTable))
currentID= dbCur.fetchone()
if currentID == None:
print "Help, couldn't insert new stream into streams table"
sys.exit(-1)
currentID =int(currentID[0])
else:
# check that this stream exists
#print streamID
dbCur.execute("select _id,hostAddress,dataTable,live,query,hostAlias,port,friendlyName from streams where _id=%d"%(streamID))
dbRow=dbCur.fetchone()
if dbRow == None:
print "Help: trying to alter a stream (id=%d) which doesn't exist"%currentID
sys.exit(-1)
currentID=dbRow[0]
# if we haven't posted a host address, then add this one
if len(hostAddress)==0:
hostAddress=dbRow[1]
#data table forces to the same table
dataTable=dbRow[2]
# liveness is read from the table if it isn't set explicitly
if live==-1:
live=int(dbRow[3])
#read in the query always - don't allow query changes without making a new stream
query=dbRow[4]
#read in the host alias always - don't allow query changes without making a new stream
hostAlias=dbRow[5]
#read in the port always - don't allow port changes without making a new stream
port=int(dbRow[6])
if len(friendlyName)==0:
friendlyName=dbRow[7]
#print "ID:",currentID
# update the stream host address, friendly name and liveness in the streams table in case they have changed and commit
with self.dbConnection:
#print (friendlyName,hostAddress,hostAlias,port,query,dataTable,live)
if len(friendlyName)>0:
self.dbConnection.execute("update streams set friendlyName=? where _id=?",(friendlyName,currentID))
if len(hostAddress)>0:
self.dbConnection.execute("update streams set hostAddress=? where _id=?",(hostAddress,currentID))
if live!=-1:
self.dbConnection.execute("update streams set live=%d where _id=%d"%(live,currentID))
# self.dbConnection.execute("update streams set friendlyName=?,hostAddress=?,live=? where _id=?",(friendlyName,hostAddress,live,currentID))
self.createdataTableIfNeeded(dataTable)
# open connection reader to ip address, port
if len(query)>0:
hostAddress=query+'@'+hostAddress
if live==1:
# see if a receiver already exists
alreadyReading=False
for c in range(0,len(self.inStreams)):
(rec,fri,tab,id,oldTime)=self.inStreams[c]
if id==currentID:
alreadyReading=True
#set a new start time
self.inStreams[c]=(rec,fri,tab,id,startTime)
if not alreadyReading:
receiver=DataReceiver(hostAddress,port, self,(dataTable,currentID,startTime))
self.inStreams.append( (receiver,friendlyName,dataTable,currentID,startTime) )
receiver.start()
print "Opened reader for %s at %s[%s]:%d"%(friendlyName,hostAlias,hostAddress,port)
else:
print "Already reading %s at %s[%s]:%d"%(friendlyName,hostAlias,hostAddress,port)
else:
foundPos=-1
foundReceiver=None
for c in range(0,len(self.inStreams)):
(rec,fri,tab,id,oldStartTime) =self.inStreams[c]
if id==currentID:
foundPos=c
foundReceiver=rec
if foundPos!=-1:
self.inStreams.pop(foundPos)
foundReceiver.stopReceiving()
