def testRecovery(self):
# print "edgeInfoRecv from fog ",edgeInfoRecv
#have to read data from edge
transport = TSocket.TSocket("127.0.0.1",5005)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TFramedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = EdgeService.Client(protocol)
# Connect!
transport.open()
response = client.read("jsr.java",1) #this is for recovery
print "Read status is ",response.status
if response.status==0 :
print "File not found : cannot read file"
else:
print "Local Read ",len(response.data)," number of bytes"
print "Metadata also read ",response.metadata
transport.close()
def openSocketConnection(self, ip, port, choice):
#print "ip ",ip," port",port," choice ", "open socket connection"
# Make socket
transport = TSocket.TSocket(ip, port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TFramedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
if (choice == FOG_SERVICE):
client = FogService.Client(protocol)
else:
#print "In the edge service call "
client = EdgeService.Client(protocol)
# Connect!
transport.open()
#print "came here"
return client, transport
def main():
# Make socket
transport = TSocket.TSocket('10.24.240.206', 8000)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TFramedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = EdgeService.Client(protocol)
# Connect!
transport.open()
# client.pong()
# print('ping()')
microbatchId = "sheshadri"
metaData = Metadata()
metaData.mbId = microbatchId
metaData.streamId = "cds"
metaData.timestamp = time.time()
myFile = open("krishna.java", 'r')
data = myFile.read()
# response = client.write(microbatchId, metaData,data)
# if(response==1):
# print "success",response
# else:
# print "failure"
readResponse = client.read(microbatchId, 0)
if (readResponse.status == 1):
print "something to read"
print "got ", len(readResponse.data), " bytes of data"
if (readResponse.metadata != None):
print "metadata requested ", readResponse.metadata
else:
print "no metadata requested "
else:
print "no read avaiblable"
# Close!
transport.close()
def writeToEdge(self, writable, microbatchID, streamId, data, EDGE_ID,
index, sizeChoice, metaData, metaKeyValueMap):
device = ""
if (writable.edgeInfo != None):
device = "local"
else:
device = str(writable.node.nodeId)
localTime = repr(time.time())
timestamp_record = str(microbatchID) + "," + str(
index
) + "," + device + "," + "write req,starttime = " + localTime + ","
#print "got the writable ",writable," the microbatchID is ",microbatchID
nodeInfo = writable.node #NodeInfoData
writePreference = writable.preference
reliability = writable.reliability
edgeInfoData = writable.edgeInfo # this is an optional field
#Write to the edge
if edgeInfoData != None:
#print "ip ",edgeInfoData.nodeIp," port",edgeInfoData.port
# Make socket
transport = TSocket.TSocket(edgeInfoData.nodeIp, edgeInfoData.port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TFramedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = EdgeService.Client(protocol)
# Connect!
transport.open()
#byte write(1:string mbId, 2:Metadata mbMetadata, 3:binary mbData)
#local write -150
timestamp_record_local = str(microbatchID) + "," + str(
-150) + ",local,write req,starttime = " + localTime + ","
#the response is not a byte anymore, its a WriteResponse
response = client.write(microbatchID, metaData, data)
timestamp_record_local = timestamp_record_local + "endtime = " + repr(
time.time()) + " , " + str(sizeChoice) + '\n'
#print "response from the edge ",response.status
transport.close()
#update the metadata structures for the Fog
client, transport = self.openSocketConnection(
nodeInfo.NodeIP, nodeInfo.port, FOG_SERVICE)
#byte insertMetadata(1: Metadata mbMetadata, 2: EdgeInfoData edgeInfoData);
#this was valid as per previous implementation in which we assumed that a local
#write means that the client will be writing to itself. However as per the new
#implementation, it is not necessary and a local write means a write that is written
#to itself or any other edge managed by the same Fog i.e. a neighbor edge of the client
#edgeInfoData = EdgeInfoData()
#edgeInfoData.nodeId = EDGE_ID
#edgeInfoData.nodeIp = EDGE_IP
#edgeInfoData.port = EDGE_PORT
#edgeInfoData.reliability = EDGE_RELIABILITY
#edgeInfoData.storage = 12 #This value is not useful for computation
#update the metadata with the checksum
#hash_md5 = hashlib.md5()
#hash_md5.update(data)
#metaData.checksum = hash_md5.hexdigest()
#metadata insert to fog -50
timeMetadata = str(microbatchID) + "," + str(
-50) + ",local ,metadata req,starttime = " + repr(
time.time()) + ","
response = client.insertMetadata(metaData, edgeInfoData,
metaKeyValueMap)
timeMetadata = timeMetadata + " endTime = " + repr(
time.time()) + " , " + str(sizeChoice) + '\n'
myLogs = open(BASE_LOG + 'logs.txt', 'a')
myLogs.write(timestamp_record_local)
myLogs.write(timeMetadata)
myLogs.close()
#print "The response from the fog is ", response
self.closeSocket(transport)
else:
#print "Have to talk to a fog with preference "
# byte write(1:Metadata mbMetadata, 2:binary data, 3:WritePreference preference);
client, transport = self.openSocketConnection(
nodeInfo.NodeIP, nodeInfo.port, FOG_SERVICE)
#response is now a WriteResponse and not a byte
response = client.putNext(metaData, data, writable.preference,
metaKeyValueMap)
#print "the response from the fog for write ",response.status
self.closeSocket(transport)
timestamp_record = timestamp_record + "endtime = " + repr(
time.time()) + " , " + str(sizeChoice) + '\n'
#print "the time stamp for write request is ",timestamp_record
myLogs = open(BASE_LOG + 'logs.txt', 'a')
myLogs.write(timestamp_record)
myLogs.close()
def findAndRead(self, microbatchId):
edgeInfoData = EdgeInfoData()
edgeInfoData.nodeId = EDGE_ID
edgeInfoData.nodeIp = EDGE_IP
edgeInfoData.port = EDGE_PORT
edgeInfoData.reliability = EDGE_RELIABILITY
edgeInfoData.storage = 12
client, transport = self.openSocketConnection(FOG_IP, FOG_PORT,
FOG_SERVICE)
timestamp_record = str(
microbatchId) + ",23, local ,find req,starttime = " + repr(
time.time()) + ","
response = client.find(microbatchId, True, True, edgeInfoData)
timestamp_record = timestamp_record + "endtime = " + repr(
time.time()) + '\n'
print("the time stamp for find request is ", timestamp_record)
myLogs = open(BASE_LOG + 'logs.txt', 'a')
myLogs.write(timestamp_record)
myLogs.close()
compFormat = str()
uncompSize = int()
## if the response contains empty list then the search is terminated here
if len(response) == 0:
print(
"Length of response = 0. Replica not found, terminating here")
return 0, 0
else:
## the microbatch is present in the system.
## for obtaining compression format and uncompressed block size. addresses the issues 1. what is the file extension?, 2. how many bytes to read from the stream.
## NOTE: this will only return a result if the block metadata is present in the fog of the current partition (i.e the read is a local read)
## This operation has a little overhead since it is only performed once. Another reason is in case it is a local read
## then an connection to an edge is directly made. But since the edge does not maintain bock metadata map, an explicit connection to
## the parent fog would have to be made once again in order to retreive the required metadata info.
## Therefore since a connection to the parent fog is already being made here it is better to make a call and retreive the indormation.
## This call is just a wild guess, it may return null. If it is supposed to be a fog read, then, anyways a connection will be made to another fog,
## we will fetch the format and size at that point of time.
## Also the fog to which the connection will be made (for a fog read) it will definitely have the corresponding block metadata
compFormatSize = client.requestCompFormatSize(microbatchId)
print(compFormatSize)
if len(compFormatSize) != 0:
## i.e format and uncompressed size present
compFormat = list(compFormatSize.keys())[0]
uncompSize = compFormatSize[compFormat]
self.closeSocket(transport)
print("Sent replicas ", response)
for findReplica in response:
edgeInfoData = findReplica.edgeInfo
if (edgeInfoData != None):
print("edgeInfoRecv from fog ", edgeInfoData)
#have to read data from edge
transport = TSocket.TSocket(edgeInfoData.nodeIp,
edgeInfoData.port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TFramedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = EdgeService.Client(protocol)
# Connect!
transport.open()
timestamp_record = str(
microbatchId) + ", " + compFormat + ", 25 , " + str(
findReplica.node.nodeId
) + " , Read req,starttime = " + repr(time.time()) + ","
response = client.read(microbatchId, 0, compFormat,
uncompSize) #this is for recovery
timestamp_record = timestamp_record + "endtime = " + repr(
time.time()) + '\n'
myLogs = open(BASE_LOG + "logs.txt", 'a')
myLogs.write(timestamp_record)
myLogs.close()
#print response
print("Read status is ", response.status)
if response.status == 0:
print("File not found : cannot read file")
return 0, 0
elif response.status == 1:
#self.formulateJsonResponse(microbatchId,response)
bytesRead = len(response.data)
print("Local Read ", len(response.data),
" number of bytes")
print("metadata also read ", response.metadata)
return 1, bytesRead #successful read
else:
return response.code, 0
transport.close()
elif (findReplica.node != None):
fogNode = findReplica.node
client, transport = self.openSocketConnection(
fogNode.NodeIP, fogNode.port, FOG_SERVICE)
timestamp_record = str(
microbatchId) + ", " + compFormat + ", 27 ," + str(
findReplica.node.nodeId
) + ",write req,starttime = " + repr(time.time()) + ","
## retreiving the compression format and the uncompressed block size for read operation from
## If you have reached here it means that the block is present in another partition and the previous
## 'client.requestCompFormatSize()' would definitely have returned null.
## (Since no block in a partition => no metadata of that block maintained by fog of that particular partition)
## Therefore fetch the format and size with the following call. This call will definitely return an entry.
compFormat = str()
uncompSize = int()
compFormatSize = client.requestCompFormatSize(microbatchId)
if len(compFormatSize) != 0:
## i.e format and uncompressed size present
compFormat = list(compFormatSize.keys())[0]
uncompSize = compFormatSize[compFormat]
response = client.read(microbatchId, 0, compFormat, uncompSize)
timestamp_record = timestamp_record + "endtime = " + repr(
time.time()) + '\n'
myLogs = open(BASE_LOG + "logs.txt", 'a')
myLogs.write(timestamp_record)
myLogs.close()
if (response.status == 1):
#self.formulateJsonResponse(microbatchId,response)
bytesRead = len(response.data)
print("Fog Amount of bytes read ", len(response.data))
return 1, bytesRead #successful read
else:
print("The queried fog does not have data")
return response.status, 0
self.closeSocket(transport)
else:
print("The queried fog does not have data")
registerEdgeDevice(EDGE_ID,EDGE_IP,EDGE_PORT,EDGE_RELIABILITY,ENCODED_SPACE)
heartbeatProcess = multiprocessing.Process(target=sendPeriodicUpdates)
heartbeatProcess.start()
# if not os.path.exists(BASE_LOG):
# os.makedirs(BASE_LOG)
# registerEdgeDevice(EDGE_ID,EDGE_IP,EDGE_PORT,EDGE_RELIABILITY,ENCODED_SPACE)
# heartbeatProcess = multiprocessing.Process(target=sendPeriodicUpdates)
# heartbeatProcess.start()
handler = EdgeServiceHandler()
processor = EdgeService.Processor(handler)
transport = TSocket.TServerSocket(host="127.0.0.1", port=8080)
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
# server =TNonblockingServer.TNonblockingServer(processor, transport)
#server =TServer.TThreadPoolServer(processor, transport, tfactory, pfactory)
#newServer = TServer.TForkingServer(processor, transport)
print('Starting the server...')
server = TServer.TThreadedServer(processor, transport, tfactory, pfactory)
server.serve()
# # You could do one of these for a multithreaded server
# # server = TServer.TThreadedServer(
# # processor, transport, tfactory, pfactory)
# # server = TServer.TThreadPoolServer(
# # processor, transport, tfactory, pfactory)
def findAndRead(self, microbatchId):
edgeInfoData = EdgeInfoData()
edgeInfoData.nodeId = EDGE_ID
edgeInfoData.nodeIp = EDGE_IP
edgeInfoData.port = EDGE_PORT
edgeInfoData.reliability = EDGE_RELIABILITY
edgeInfoData.storage = 12
client,transport = self.openSocketConnection(FOG_IP,FOG_PORT,FOG_SERVICE)
timestamp_record = microbatchId+ ",23, local ,find req,starttime = "+str(time.time())+","
response = client.find(microbatchId,True,True,edgeInfoData)
timestamp_record = timestamp_record +"endtime = " + str(time.time()) + '\n'
print "the time stamp for find request is ",timestamp_record
myLogs = open(BASE_LOG+ 'logs.txt','a')
myLogs.write(timestamp_record)
myLogs.close()
self.closeSocket(transport)
print "Sent replicas ",response
for findReplica in response :
edgeInfoData = findReplica.edgeInfo
if(edgeInfoData!=None):
print "edgeInfoRecv from fog ",edgeInfoData
#have to read data from edge
transport = TSocket.TSocket(edgeInfoData.nodeIp,edgeInfoData.port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TFramedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = EdgeService.Client(protocol)
# Connect!
transport.open()
timestamp_record = microbatchId+", 25 , "+ str(findReplica.node.nodeId) + " , Read req,starttime = "+str(time.time())+","
response = client.read(microbatchId,0) #this is for recovery
timestamp_record = timestamp_record +"endtime = " + str(time.time()) + '\n'
myLogs = open(BASE_LOG+ "logs.txt",'a')
myLogs.write(timestamp_record)
myLogs.close()
print "Read status is ",response.status
if response.status==0 :
print "File not found : cannot read file"
else:
print "Local Read ",len(response.data)," number of bytes"
print "Metadata also read ",response.metadata
return 1 #successful read
transport.close()
elif(findReplica.node!=None) :
fogNode = findReplica.node
client,transport = self.openSocketConnection(fogNode.NodeIP,fogNode.port,FOG_SERVICE)
timestamp_record = microbatchId+", 27 ,"+str(findReplica.node.nodeId) + ",write req,starttime = "+str(time.time())+","
response = client.read(microbatchId,0)
timestamp_record = timestamp_record +"endtime = " + str(time.time()) + '\n'
myLogs = open(BASE_LOG+ "logs.txt",'a')
myLogs.write(timestamp_record)
myLogs.close()
if(response.status == 1):
print "Fog Amount of bytes read ",len(response.data)
return 1 #successful read
else:
print "The queried fog does not have data"
self.closeSocket(transport)
else:
print "The queried fog does not have data"
def readFromEdgeOrFog(self,microbatchID):
client,transport = self.openSocketConnection(FOG_IP,FOG_PORT,FOG_SERVICE)
# ReadResponse read(1: string microbatchId, 2:bool checkLocal, 3:bool checkNeighbors, 4:bool checkBuddies, 5:EdgeInfoData selfInfo);
edgeInfoData = EdgeInfoData()
edgeInfoData.nodeId = EDGE_ID
edgeInfoData.nodeIp = EDGE_IP
edgeInfoData.port = EDGE_PORT
edgeInfoData.reliability = EDGE_RELIABILITY
edgeInfoData.storage = 12
timestamp_record = microbatchID+",read req,starttime = "+str(time.time())+","
response = client.read(microbatchID,True,True,True,edgeInfoData,True)#last bit is for recovery
timestamp_record = timestamp_record +"endtime = " + str(time.time()) + '\n'
myLogs = open(BASE_LOG+ 'logs.txt','a')
myLogs.write(timestamp_record)
myLogs.close()
if(response.status == 0):
print "File not found : cannot read file"
else:
edgeInfoRecv = response.edgeInfo
if(edgeInfoRecv!=None):
print "edgeInfoRecv from fog ",edgeInfoRecv
#have to read data from edge
transport = TSocket.TSocket(edgeInfoRecv.nodeIp,edgeInfoRecv.port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TFramedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = EdgeService.Client(protocol)
# Connect!
transport.open()
response = client.read(microbatchID,1) #this is for recovery
print "Read status is ",response.status
if response.status==0 :
print "File not found : cannot read file"
else:
print "Local Read ",len(response.data)," number of bytes"
print "Metadata also read ",response.metadata
transport.close()
else:
if response.status == 0:
print "File not found : cannot find file"
else:
print "Remote Read ",len(response.data), " number of bytes"
print "Metadata also read ",response.metadata
self.closeSocket(transport)