def processNode(nodeExists):
if nodeExists:
return Mercury.updateAndGetNode(reply['NodeId'], ip,
reply['ListeningPort'])
# This returns deferred
else:
return Mercury.createNode(reply['NodeId'], ip,
reply['ListeningPort'])
def receiveNewSyncTimestamp(self, NewSyncTimestamp):
newSyncTimestamp = NewSyncTimestamp
cprint('FROM REMOTE: NEW REMOTE TIMESTAMP: %s' % newSyncTimestamp,
'white',
'on_yellow',
attrs=['bold'])
print('\n')
Mercury.insertNewSyncTimestamp(self.connectedNode, newSyncTimestamp)
return {}
def respondToHandshake(self, NodeId, ListeningPort, ProtocolVersion):
ip = self.transport.getPeer().host
d = Mercury.checkIfNodeExists(NodeId)
def processNode(nodeExists):
if nodeExists:
return Mercury.updateAndGetNode(NodeId, ip, ListeningPort) # This returns deferred
else:
return Mercury.createNode(NodeId, ip, ListeningPort)
d.addCallback(processNode)
def callReverseSync(node):
# I'm going to ask for nodes in parallel with already existing header call.
d = self.requestHeaders(node)
d.addCallback(self.requestNodes)
return node
d.addCallback(callReverseSync)\
.addErrback(self.closeConnection, 'INBOUND', callReverseSync.__name__)
d.addCallback(self.setGlobalConnectedNode) # This is at one level up, directly below protocol class.
reply = {'NodeId': self.factory.localNodeId,
'ListeningPort': aetherListeningPort,
'ProtocolVersion': protocolVersion }
cprint('FROM REMOTE: HANDSHAKE REQUEST: from %s:%s'
%(ip, ListeningPort), 'white', 'on_yellow', attrs=['bold'])
cprint('ANSWER: %s' %(reply), 'white', 'on_yellow')
print('\n')
return reply
def respondWithNodes(self, LastSyncTimestamp):
if not self.connectedNode:
self.abortConnection(
Exception('AetherError: Illegal request from remote.'), 'L<-R',
'respondWithNodes')
if LastSyncTimestamp != 'null':
LastSyncTimestamp = datetime.utcfromtimestamp(
float(LastSyncTimestamp))
else:
LastSyncTimestamp = 'NO-TIMESTAMP'
print(
'L<-R: Nodes request, Timestamp: ', LastSyncTimestamp
if LastSyncTimestamp != 'NO-TIMESTAMP' else 'Initial Connection',
' N:', self.connectedNode['NodeId'])
d = Mercury.getNodes(LastSyncTimestamp)
d.addCallback(Mercury.processNodes)
currentPacketNumber = 1
totalPacketNumber = 1
def calculatePacketCounts(dbReply, bucketSize):
global totalPacketNumber
totalPacketNumber = len(dbReply) / bucketSize
if len(dbReply) % bucketSize > 0:
totalPacketNumber += 1
if len(dbReply) == 0:
totalPacketNumber = 1
return dbReply
d.addCallback(calculatePacketCounts, NODE_PACKET_COUNT)
def bucketise(dbReply, bucketSize, CPNo):
# Here I have the entire reply from the db. I need divide that reply into buckets, and fire each
# bucket to the remote.
def process(slicedDbReply, currentPacketNumber):
print('L->R: Nodes reply, Package number: ',
currentPacketNumber, ' N:', self.connectedNode['NodeId'])
global totalPacketNumber
nodesJson = []
for n in slicedDbReply:
nJson = ujson.dumps(n)
nodesJson.append(nJson)
self.callRemote(networkAPI.ReceiveNodes,
Nodes=nodesJson,
TotalNumberOfPackets=totalPacketNumber,
CurrentPacketNo=currentPacketNumber)
return {}
if len(dbReply) > bucketSize:
bucketise(dbReply[bucketSize:], bucketSize,
CPNo + 1) # next batch.
return process(dbReply[0:bucketSize], CPNo) # current batch.
else:
return process(dbReply[0:bucketSize], CPNo) # current batch.
d.addCallback(bucketise, NODE_PACKET_COUNT, currentPacketNumber)
d.addErrback(self.abortConnection, 'L<-R',
self.respondWithNodes.__name__)
return d
def replyArrived(reply):
self.connectionState = 'HANDSHAKE'
ip = self.transport.getPeer().host
d = Mercury.handleConnectingNode(reply['NodeId'], ip, reply['ListeningPort']) # returns a node as dict.
# This sets protocol globals that only become available after node is ID'd.
d.addCallback(self.postHandshakeInitialization)
d.addCallback(self.requestHeaders)
print('L<-R: Handshake reply. N: ', reply['NodeId'])
def respondWithNodes(self, LastSyncTimestamp):
if not self.connectedNode:
self.abortConnection(Exception("AetherError: Illegal request from remote."), "L<-R", "respondWithNodes")
if LastSyncTimestamp != "null":
LastSyncTimestamp = datetime.utcfromtimestamp(float(LastSyncTimestamp))
else:
LastSyncTimestamp = "NO-TIMESTAMP"
print(
"L<-R: Nodes request, Timestamp: ",
LastSyncTimestamp if LastSyncTimestamp != "NO-TIMESTAMP" else "Initial Connection",
" N:",
self.connectedNode["NodeId"],
)
d = Mercury.getNodes(LastSyncTimestamp)
d.addCallback(Mercury.processNodes)
currentPacketNumber = 1
totalPacketNumber = 1
def calculatePacketCounts(dbReply, bucketSize):
global totalPacketNumber
totalPacketNumber = len(dbReply) / bucketSize
if len(dbReply) % bucketSize > 0:
totalPacketNumber += 1
if len(dbReply) == 0:
totalPacketNumber = 1
return dbReply
d.addCallback(calculatePacketCounts, NODE_PACKET_COUNT)
def bucketise(dbReply, bucketSize, CPNo):
# Here I have the entire reply from the db. I need divide that reply into buckets, and fire each
# bucket to the remote.
def process(slicedDbReply, currentPacketNumber):
print("L->R: Nodes reply, Package number: ", currentPacketNumber, " N:", self.connectedNode["NodeId"])
global totalPacketNumber
nodesJson = []
for n in slicedDbReply:
nJson = ujson.dumps(n)
nodesJson.append(nJson)
self.callRemote(
networkAPI.ReceiveNodes,
Nodes=nodesJson,
TotalNumberOfPackets=totalPacketNumber,
CurrentPacketNo=currentPacketNumber,
)
return {}
if len(dbReply) > bucketSize:
bucketise(dbReply[bucketSize:], bucketSize, CPNo + 1) # next batch.
return process(dbReply[0:bucketSize], CPNo) # current batch.
else:
return process(dbReply[0:bucketSize], CPNo) # current batch.
d.addCallback(bucketise, NODE_PACKET_COUNT, currentPacketNumber)
d.addErrback(self.abortConnection, "L<-R", self.respondWithNodes.__name__)
return d
def receiveHeaderPacket(self, PositiveHeaders, NeutralHeaders,
NegativeHeaders, TopicHeaders,
TotalNumberOfPackets, CurrentPacketNo):
if not self.connectedNode:
self.abortConnection(
Exception('AetherError: Illegal request from remote.'), 'L<-R',
'receiveHeaderPacket')
print('L<-R: Header reply, Package number: ', CurrentPacketNo, ' N:',
self.connectedNode['NodeId'])
# Define what just arrived from the network.
reply = {
'PositiveHeaders': PositiveHeaders,
'NeutralHeaders': NeutralHeaders,
'NegativeHeaders': NegativeHeaders,
'TopicHeaders': TopicHeaders,
'TotalNumberOfPackets': TotalNumberOfPackets,
'CurrentPacketNo': CurrentPacketNo
}
# Set the global state machine information.
self.lastArrival = datetime.utcnow()
self.connectionState = 'HEADER'
# Set the local data about this state's status.
self.expectedHeaderPackets = reply['TotalNumberOfPackets']
# Things that need to happen with every packet.
d = Mercury.insertHeadersAndVotes(reply, self.connectedNode)
d.addCallback(Mercury.listNeededPosts
) # Returns: an array of needed posts' fingerprints.
def saveNeededPostsList(currentPacketNeededPosts):
#print('current packet needed posts: ', currentPacketNeededPosts)
# for postFingerprint in currentPacketNeededPosts:
# if postFingerprint not in self.neededPosts:
# self.neededPosts.append(postFingerprint)
# self.expectedPosts += 1
# I don't need to do this check, because I am doing this kind of check at the commit phase already.
self.neededPosts += currentPacketNeededPosts
self.expectedPosts += len(currentPacketNeededPosts)
# Set the local data about this state's status.
self.arrivedHeaderPackets += 1
self.lastHeaderPacketArrival = datetime.utcnow()
# I moved them here from one outer scope, because when the remote has a million posts, listneededposts
# take much longer than header arrival. as checkifdone is now async, checkifdone actually triggers
# much later than header arrival, unlike intended use of the last one triggering it, almost all triggers.
d.addCallback(saveNeededPostsList)
# Things that need to happen only after all header posts have arrived.
def checkIfDone(*a):
if self.expectedHeaderPackets == self.arrivedHeaderPackets:
self.advanceToNextStateFromHeader()
# This is a callback because before I did that it was triggering before everything was actually appended.
d.addCallback(checkIfDone)
return {}
def receiveHeaderPacket(self, PositiveHeaders, NeutralHeaders,
NegativeHeaders, TopicHeaders,
TotalNumberOfPackets, CurrentPacketNo):
cprint('RECEIVE HEADER PACKET RESPONDER WAS CALLED.', 'white',
'on_red')
# This is just a proof of concept, so I'm mushing all the answers together to create the reply replyArrived
# (below) expects.
self.headerTransferStarted = True
# Here, I assign the totalnumber to the protocol and start counting from that.
self.totalNumberOfHeaderPackets = TotalNumberOfPackets
print('total number of header packets: ',
self.totalNumberOfHeaderPackets)
print('current packet count: ', self.currentHeaderPacketCount)
reply = {
'PositiveHeaders': PositiveHeaders,
'NeutralHeaders': NeutralHeaders,
'NegativeHeaders': NegativeHeaders,
'TopicHeaders': TopicHeaders,
'TotalNumberOfPackets': TotalNumberOfPackets,
'CurrentPacketNo': CurrentPacketNo
}
node = self.connectedNode
# I have two tasks, the first is to insert appropriate votes,
# the second is to decide on which posts I will request.
d = Mercury.insertHeaders(reply)
d.addCallback(Mercury.insertVotes, node)
d.addCallback(Mercury.listNeededPosts
) # this will return an array of needed posts.
def decrementAndCheckPostCounter(*args):
setattr(self, 'incomingPostCountForHeaderPacket'+str(CurrentPacketNo),
getattr(self, 'incomingPostCountForHeaderPacket'+str(CurrentPacketNo)) - 1) if \
getattr(self, 'incomingPostCountForHeaderPacket'+str(CurrentPacketNo)) != 0 else 0
# Here is how I figure out the connection is completed.
if getattr(
self, 'incomingPostCountForHeaderPacket' +
str(CurrentPacketNo)) == 0:
print('incoming posts of header %s is completed' %
CurrentPacketNo)
# This only enters after the posts of the header packet given complete transmitting.
# This is the final point of process of a header and this marks this header packet done.
self.currentHeaderPacketCount += 1
def firePostRequests(neededPosts):
setattr(self,
'incomingPostCountForHeaderPacket' + str(CurrentPacketNo),
len(neededPosts))
for fp in neededPosts:
d2 = self.requestPost(fp)
d2.addCallback(decrementAndCheckPostCounter)
if len(neededPosts) == 0:
decrementAndCheckPostCounter()
d.addCallback(firePostRequests)
return node
def receiveNodePacket(self, Nodes, TotalNumberOfPackets, CurrentPacketNo):
if not self.connectedNode:
self.abortConnection(Exception('AetherError: Illegal request from remote.'), 'L<-R', 'receiveNodePacket')
print('L<-R: Nodes reply, Package number: ', CurrentPacketNo, ' N:', self.connectedNode['NodeId'])
# Set the global state machine information.
self.lastArrival = datetime.utcnow()
self.connectionState = 'NODE'
# Set the local data about this state's status.
self.expectedNodePackets = TotalNumberOfPackets
self.arrivedNodePackets += 1
# Things that need to happen at every node packet arrival.
Mercury.insertNodes(Nodes, self.factory.localNodeId)
# Things that need to happen only after all node packets arrive.
if self.expectedNodePackets == self.arrivedNodePackets:
self.advanceToNextStateFromNode()
return {}
def replyArrived(reply):
self.connectionState = 'HANDSHAKE'
ip = self.transport.getPeer().host
d = Mercury.handleConnectingNode(
reply['NodeId'], ip,
reply['ListeningPort']) # returns a node as dict.
# This sets protocol globals that only become available after node is ID'd.
d.addCallback(self.postHandshakeInitialization)
d.addCallback(self.requestHeaders)
print('L<-R: Handshake reply. N: ', reply['NodeId'])
def respondWithPost(self, PostFingerprint):
d = Mercury.getPost(PostFingerprint)
d.addCallback(Mercury.toJson)
def processAndReturn(postAsJson):
reply = {'Post': postAsJson}
return reply
d.addCallback(processAndReturn)\
.addErrback(self.closeConnection, 'INBOUND', self.respondWithPost.__name__)
return d
def respondWithPost(self, PostFingerprint):
d = Mercury.getPost(PostFingerprint)
d.addCallback(Mercury.toJson)
def processAndReturn(postAsJson):
reply = { 'Post': postAsJson }
return reply
d.addCallback(processAndReturn)\
.addErrback(self.closeConnection, 'INBOUND', self.respondWithPost.__name__)
return d
def receiveHeaderPacket(self, PositiveHeaders, NeutralHeaders, NegativeHeaders,
TopicHeaders, TotalNumberOfPackets, CurrentPacketNo):
if not self.connectedNode:
self.abortConnection(Exception('AetherError: Illegal request from remote.'), 'L<-R', 'receiveHeaderPacket')
print('L<-R: Header reply, Package number: ', CurrentPacketNo, ' N:', self.connectedNode['NodeId'])
# Define what just arrived from the network.
reply = {
'PositiveHeaders': PositiveHeaders,
'NeutralHeaders': NeutralHeaders,
'NegativeHeaders': NegativeHeaders,
'TopicHeaders': TopicHeaders,
'TotalNumberOfPackets': TotalNumberOfPackets,
'CurrentPacketNo': CurrentPacketNo
}
# Set the global state machine information.
self.lastArrival = datetime.utcnow()
self.connectionState = 'HEADER'
# Set the local data about this state's status.
self.expectedHeaderPackets = reply['TotalNumberOfPackets']
# Things that need to happen with every packet.
d = Mercury.insertHeadersAndVotes(reply, self.connectedNode)
d.addCallback(Mercury.listNeededPosts) # Returns: an array of needed posts' fingerprints.
def saveNeededPostsList(currentPacketNeededPosts):
#print('current packet needed posts: ', currentPacketNeededPosts)
# for postFingerprint in currentPacketNeededPosts:
# if postFingerprint not in self.neededPosts:
# self.neededPosts.append(postFingerprint)
# self.expectedPosts += 1
# I don't need to do this check, because I am doing this kind of check at the commit phase already.
self.neededPosts += currentPacketNeededPosts
self.expectedPosts += len(currentPacketNeededPosts)
# Set the local data about this state's status.
self.arrivedHeaderPackets += 1
self.lastHeaderPacketArrival = datetime.utcnow()
# I moved them here from one outer scope, because when the remote has a million posts, listneededposts
# take much longer than header arrival. as checkifdone is now async, checkifdone actually triggers
# much later than header arrival, unlike intended use of the last one triggering it, almost all triggers.
d.addCallback(saveNeededPostsList)
# Things that need to happen only after all header posts have arrived.
def checkIfDone(*a):
if self.expectedHeaderPackets == self.arrivedHeaderPackets:
self.advanceToNextStateFromHeader()
# This is a callback because before I did that it was triggering before everything was actually appended.
d.addCallback(checkIfDone)
return {}
def replyArrived(reply):
print('L<-R: Post reply, Fingerprint: ', fingerprint, ' N:', self.connectedNode['NodeId'])
# Things that happen at every arrival.
self.lastArrival = datetime.utcnow()
self.lastPostArrival = datetime.utcnow()
self.arrivedPosts += 1
self.connectionState = 'POST'
d = Mercury.insertPost(reply)
# Things that happen only after all posts arrive.
if self.expectedPosts == self.arrivedPosts:
self.advanceToNextStateFromPost()
return d
def respondWithPost(self, PostFingerprint):
if not self.connectedNode:
self.abortConnection(Exception("AetherError: Illegal request from remote."), "L<-R", "respondWithPost")
print("L->R: Post reply. Post: ", PostFingerprint, " N:", self.connectedNode["NodeId"])
d = Mercury.getPost(PostFingerprint)
d.addCallback(Mercury.toJson)
def processAndReturn(postAsJson):
reply = {"Post": postAsJson}
return reply
d.addCallback(processAndReturn).addErrback(self.abortConnection, "L<-R", self.respondWithPost.__name__)
return d
def receiveNodePacket(self, Nodes, TotalNumberOfPackets, CurrentPacketNo):
cprint('RECEIVE NODE PACKET RESPONDER WAS CALLED.', 'white', 'on_yellow')
self.totalNumberOfNodePackets = TotalNumberOfPackets
print('total number of NODE packets: ',self.totalNumberOfNodePackets)
print('current NODE packet count: ', self.currentNodePacketCount)
self.nodeTransferStarted = True
d= Mercury.insertNodes(Nodes, self.factory.localNodeId)
def incrementNodePacketCount(ignored):
self.currentNodePacketCount += 1
d.addCallback(incrementNodePacketCount)
return {}
def receiveNodePacket(self, Nodes, TotalNumberOfPackets, CurrentPacketNo):
if not self.connectedNode:
self.abortConnection(
Exception('AetherError: Illegal request from remote.'), 'L<-R',
'receiveNodePacket')
print('L<-R: Nodes reply, Package number: ', CurrentPacketNo, ' N:',
self.connectedNode['NodeId'])
# Set the global state machine information.
self.lastArrival = datetime.utcnow()
self.connectionState = 'NODE'
# Set the local data about this state's status.
self.expectedNodePackets = TotalNumberOfPackets
self.arrivedNodePackets += 1
# Things that need to happen at every node packet arrival.
Mercury.insertNodes(Nodes, self.factory.localNodeId)
# Things that need to happen only after all node packets arrive.
if self.expectedNodePackets == self.arrivedNodePackets:
self.advanceToNextStateFromNode()
return {}
def replyArrived(reply):
print('L<-R: Post reply, Fingerprint: ', fingerprint, ' N:',
self.connectedNode['NodeId'])
# Things that happen at every arrival.
self.lastArrival = datetime.utcnow()
self.lastPostArrival = datetime.utcnow()
self.arrivedPosts += 1
self.connectionState = 'POST'
d = Mercury.insertPost(reply)
# Things that happen only after all posts arrive.
if self.expectedPosts == self.arrivedPosts:
self.advanceToNextStateFromPost()
return d
def respondWithPost(self, PostFingerprint):
if not self.connectedNode:
self.abortConnection(Exception('AetherError: Illegal request from remote.'), 'L<-R', 'respondWithPost')
print('L->R: Post reply. Post: ', PostFingerprint, ' N:', self.connectedNode['NodeId'])
d = Mercury.getPost(PostFingerprint)
d.addCallback(Mercury.toJson)
def processAndReturn(postAsJson):
reply = { 'Post': postAsJson }
return reply
d.addCallback(processAndReturn)\
.addErrback(self.abortConnection, 'L<-R', self.respondWithPost.__name__)
return d
def receiveHeaderPacket(self, PositiveHeaders, NeutralHeaders, NegativeHeaders,
TopicHeaders, TotalNumberOfPackets, CurrentPacketNo):
cprint('RECEIVE HEADER PACKET RESPONDER WAS CALLED.', 'white', 'on_red')
# This is just a proof of concept, so I'm mushing all the answers together to create the reply replyArrived
# (below) expects.
self.headerTransferStarted = True
# Here, I assign the totalnumber to the protocol and start counting from that.
self.totalNumberOfHeaderPackets = TotalNumberOfPackets
print('total number of header packets: ',self.totalNumberOfHeaderPackets)
print('current packet count: ', self.currentHeaderPacketCount)
reply = {
'PositiveHeaders': PositiveHeaders,
'NeutralHeaders': NeutralHeaders,
'NegativeHeaders': NegativeHeaders,
'TopicHeaders': TopicHeaders,
'TotalNumberOfPackets': TotalNumberOfPackets,
'CurrentPacketNo': CurrentPacketNo
}
node = self.connectedNode
# I have two tasks, the first is to insert appropriate votes,
# the second is to decide on which posts I will request.
d = Mercury.insertHeaders(reply)
d.addCallback(Mercury.insertVotes, node)
d.addCallback(Mercury.listNeededPosts) # this will return an array of needed posts.
def decrementAndCheckPostCounter(*args):
setattr(self, 'incomingPostCountForHeaderPacket'+str(CurrentPacketNo),
getattr(self, 'incomingPostCountForHeaderPacket'+str(CurrentPacketNo)) - 1) if \
getattr(self, 'incomingPostCountForHeaderPacket'+str(CurrentPacketNo)) != 0 else 0
# Here is how I figure out the connection is completed.
if getattr(self, 'incomingPostCountForHeaderPacket'+str(CurrentPacketNo)) == 0:
print('incoming posts of header %s is completed'%CurrentPacketNo)
# This only enters after the posts of the header packet given complete transmitting.
# This is the final point of process of a header and this marks this header packet done.
self.currentHeaderPacketCount += 1
def firePostRequests(neededPosts):
setattr(self, 'incomingPostCountForHeaderPacket'+str(CurrentPacketNo), len(neededPosts))
for fp in neededPosts:
d2 = self.requestPost(fp)
d2.addCallback(decrementAndCheckPostCounter)
if len(neededPosts) == 0:
decrementAndCheckPostCounter()
d.addCallback(firePostRequests)
return node
def replyArrived(reply):
cprint('RECEIVED: HANDSHAKE REPLY. \n%s' %(reply), 'cyan', 'on_blue')
print('\n')
ip = self.transport.getPeer().host
d = Mercury.checkIfNodeExists(reply['NodeId'])
def processNode(nodeExists):
if nodeExists:
return Mercury.updateAndGetNode(reply['NodeId'], ip, reply['ListeningPort'])
# This returns deferred
else:
return Mercury.createNode(reply['NodeId'], ip, reply['ListeningPort'])
d.addCallback(processNode)
d.addCallback(self.setGlobalConnectedNode) # This is at one level up, directly below protocol class.
d.addCallback(self.requestHeaders)
d.addCallback(self.requestNodes)
def receiveNodePacket(self, Nodes, TotalNumberOfPackets, CurrentPacketNo):
cprint('RECEIVE NODE PACKET RESPONDER WAS CALLED.', 'white',
'on_yellow')
self.totalNumberOfNodePackets = TotalNumberOfPackets
print('total number of NODE packets: ', self.totalNumberOfNodePackets)
print('current NODE packet count: ', self.currentNodePacketCount)
self.nodeTransferStarted = True
d = Mercury.insertNodes(Nodes, self.factory.localNodeId)
def incrementNodePacketCount(ignored):
self.currentNodePacketCount += 1
d.addCallback(incrementNodePacketCount)
return {}
def respondWithPost(self, PostFingerprint):
if not self.connectedNode:
self.abortConnection(
Exception('AetherError: Illegal request from remote.'), 'L<-R',
'respondWithPost')
print('L->R: Post reply. Post: ', PostFingerprint, ' N:',
self.connectedNode['NodeId'])
d = Mercury.getPost(PostFingerprint)
d.addCallback(Mercury.toJson)
def processAndReturn(postAsJson):
reply = {'Post': postAsJson}
return reply
d.addCallback(processAndReturn)\
.addErrback(self.abortConnection, 'L<-R', self.respondWithPost.__name__)
return d
def respondToHandshake(self, NodeId, ListeningPort, ProtocolVersion):
print('L<-R: Handshake request from ', self.transport.getPeer().host, ' N:', NodeId)
self.connectionState = 'HANDSHAKE'
ip = self.transport.getPeer().host
if len(self.factory.openConnections) > globals.maxInboundCount:
# cprint('Connection refused because I\'m too busy.', 'red')
self.transport.loseConnection()
else:
d = Mercury.handleConnectingNode(NodeId, ip, ListeningPort) # returns a node as dict.
d.addCallback(self.postHandshakeInitialization)
# Reverse sync here. I'm asking the remote about his contents.
d.addCallback(self.requestHeaders)\
.addErrback(self.abortConnection, 'L<-R', self.requestHeaders.__name__)
reply = {'NodeId': self.factory.localNodeId,
'ListeningPort': aetherListeningPort,
'ProtocolVersion': protocolVersion }
print('L->R: Handshake reply sent.', ' N:', NodeId)
return reply
def respondToHandshake(self, NodeId, ListeningPort, ProtocolVersion):
print("L<-R: Handshake request from ", self.transport.getPeer().host, " N:", NodeId)
self.connectionState = "HANDSHAKE"
ip = self.transport.getPeer().host
if len(self.factory.openConnections) > globals.userProfile.get("machineDetails", "maxInboundCount"):
print("Connection refused because I'm too busy.")
self.transport.loseConnection()
else:
d = Mercury.handleConnectingNode(NodeId, ip, ListeningPort) # returns a node as dict.
d.addCallback(self.postHandshakeInitialization)
# Reverse sync here. I'm asking the remote about his contents.
d.addCallback(self.requestHeaders).addErrback(self.abortConnection, "L<-R", self.requestHeaders.__name__)
reply = {
"NodeId": self.factory.localNodeId,
"ListeningPort": aetherListeningPort,
"ProtocolVersion": PROT_VERSION,
}
print("L->R: Handshake reply sent.", " N:", NodeId)
return reply
def replyArrived(reply):
cprint('RECEIVED: HANDSHAKE REPLY. \n%s' % (reply), 'cyan',
'on_blue')
print('\n')
ip = self.transport.getPeer().host
d = Mercury.checkIfNodeExists(reply['NodeId'])
def processNode(nodeExists):
if nodeExists:
return Mercury.updateAndGetNode(reply['NodeId'], ip,
reply['ListeningPort'])
# This returns deferred
else:
return Mercury.createNode(reply['NodeId'], ip,
reply['ListeningPort'])
d.addCallback(processNode)
d.addCallback(
self.setGlobalConnectedNode
) # This is at one level up, directly below protocol class.
d.addCallback(self.requestHeaders)
d.addCallback(self.requestNodes)
def respondToHandshake(self, NodeId, ListeningPort, ProtocolVersion):
print('L<-R: Handshake request from ',
self.transport.getPeer().host, ' N:', NodeId)
self.connectionState = 'HANDSHAKE'
ip = self.transport.getPeer().host
if len(self.factory.openConnections) > globals.maxInboundCount:
# cprint('Connection refused because I\'m too busy.', 'red')
self.transport.loseConnection()
else:
d = Mercury.handleConnectingNode(
NodeId, ip, ListeningPort) # returns a node as dict.
d.addCallback(self.postHandshakeInitialization)
# Reverse sync here. I'm asking the remote about his contents.
d.addCallback(self.requestHeaders)\
.addErrback(self.abortConnection, 'L<-R', self.requestHeaders.__name__)
reply = {
'NodeId': self.factory.localNodeId,
'ListeningPort': aetherListeningPort,
'ProtocolVersion': protocolVersion
}
print('L->R: Handshake reply sent.', ' N:', NodeId)
return reply
def respondToHandshake(self, NodeId, ListeningPort, ProtocolVersion):
ip = self.transport.getPeer().host
d = Mercury.checkIfNodeExists(NodeId)
def processNode(nodeExists):
if nodeExists:
return Mercury.updateAndGetNode(
NodeId, ip, ListeningPort) # This returns deferred
else:
return Mercury.createNode(NodeId, ip, ListeningPort)
d.addCallback(processNode)
def callReverseSync(node):
# I'm going to ask for nodes in parallel with already existing header call.
d = self.requestHeaders(node)
d.addCallback(self.requestNodes)
return node
d.addCallback(callReverseSync)\
.addErrback(self.closeConnection, 'INBOUND', callReverseSync.__name__)
d.addCallback(
self.setGlobalConnectedNode
) # This is at one level up, directly below protocol class.
reply = {
'NodeId': self.factory.localNodeId,
'ListeningPort': aetherListeningPort,
'ProtocolVersion': protocolVersion
}
cprint('FROM REMOTE: HANDSHAKE REQUEST: from %s:%s' %
(ip, ListeningPort),
'white',
'on_yellow',
attrs=['bold'])
cprint('ANSWER: %s' % (reply), 'white', 'on_yellow')
print('\n')
return reply
def processNode(nodeExists):
if nodeExists:
return Mercury.updateAndGetNode(reply['NodeId'], ip, reply['ListeningPort'])
# This returns deferred
else:
return Mercury.createNode(reply['NodeId'], ip, reply['ListeningPort'])
def respondWithNodes(self, LastSyncTimestamp):
if LastSyncTimestamp != 'null':
LastSyncTimestamp = datetime.utcfromtimestamp(
float(LastSyncTimestamp)) # + timedelta(minutes = 1)
else:
LastSyncTimestamp = 'NO-TIMESTAMP'
d = Mercury.getNodes(LastSyncTimestamp)
d.addCallback(Mercury.processNodes)
cprint('FROM REMOTE: NODES REQUEST: with sync timestamp %s' %
(LastSyncTimestamp
if LastSyncTimestamp != 'NO-TIMESTAMP' else LastSyncTimestamp),
'white',
'on_yellow',
attrs=['bold'])
print('\n')
currentPacketNumber = 1
totalPacketNumber = 1
def calculatePacketCounts(dbReply, bucketSize):
global totalPacketNumber
totalPacketNumber = len(dbReply) / bucketSize
if len(dbReply) % bucketSize > 0:
totalPacketNumber += 1
if len(dbReply) == 0:
totalPacketNumber = 1
print('the reply includes %s headers' % len(dbReply))
print('total packet numbers: %s' % totalPacketNumber)
return dbReply
d.addCallback(calculatePacketCounts, globals.nodePacketCount)
def bucketise(dbReply, bucketSize, CPNo):
# Here I have the entire reply from the db. I need divide that reply into buckets, and fire each
# bucket to the remote.
def process(slicedDbReply, currentPacketNumber):
global totalPacketNumber
nodesJson = []
for n in slicedDbReply:
nJson = ujson.dumps(n)
nodesJson.append(nJson)
cprint(
'THIS IS NODE ANSWER PACKAGE NUMBERED: %d' %
currentPacketNumber, 'white', 'on_red')
print(
'reply: ', {
'Nodes': nodesJson,
'TotalNumberOfPackets': totalPacketNumber,
'CurrentPacketNo': currentPacketNumber
})
# here I need to fire receiveheaders methods onto the remote for each of the packets.
self.callRemote(networkAPI.ReceiveNodes,
Nodes=nodesJson,
TotalNumberOfPackets=totalPacketNumber,
CurrentPacketNo=currentPacketNumber)
return {}
if len(dbReply) > bucketSize:
bucketise(dbReply[bucketSize:], bucketSize,
CPNo + 1) # next batch.
return process(dbReply[0:bucketSize], CPNo) # current batch.
else:
return process(dbReply[0:bucketSize], CPNo) # current batch.
d.addCallback(bucketise, globals.nodePacketCount, currentPacketNumber)
d.addErrback(self.closeConnection, 'INBOUND',
self.respondWithNodes.__name__)
return d
def respondWithHeaders(self, LastSyncTimestamp, Languages):
#Check if handshake actually occurred. If not, abort.
if not self.connectedNode:
self.abortConnection(
Exception('AetherError: Illegal request from remote.'), 'L<-R',
'respondWithHeaders')
return {}
if LastSyncTimestamp != 'null':
LastSyncTimestamp = datetime.utcfromtimestamp(
float(LastSyncTimestamp))
else:
LastSyncTimestamp = 'NO-TIMESTAMP'
print(
'L<-R: Headers request, Timestamp: ', LastSyncTimestamp
if LastSyncTimestamp != 'NO-TIMESTAMP' else 'Initial Connection',
' N:', self.connectedNode['NodeId'])
d = Mercury.getHeaders(LastSyncTimestamp, Languages)
currentPacketNumber = 1
totalPacketNumber = 1 # start from one.
def calculatePacketCounts(dbReply, bucketSize):
global totalPacketNumber
totalPacketNumber = len(dbReply) / bucketSize
if len(dbReply) % bucketSize > 0:
totalPacketNumber += 1
if len(dbReply) == 0:
totalPacketNumber = 1
return dbReply
d.addCallback(calculatePacketCounts, HEADER_PACKET_COUNT)
def bucketise(dbReply, bucketSize, CPNo):
def process(slicedDbReply, currentPacketNumber):
# Here I need to fire the bucket given.
def construct(bucket):
positiveHeadersArray = []
neutralHeadersArray = []
negativeHeadersArray = []
topicHeadersArray = []
for h in bucket:
if h['DIRECTION'] == 'POSITIVE':
del h['DIRECTION']
hJson = ujson.dumps(h)
positiveHeadersArray.append(hJson)
elif h['DIRECTION'] == 'NEUTRAL':
del h['DIRECTION']
hJson = ujson.dumps(h)
neutralHeadersArray.append(hJson)
elif h['DIRECTION'] == 'NEGATIVE':
del h['DIRECTION']
hJson = ujson.dumps(h)
negativeHeadersArray.append(hJson)
elif h['DIRECTION'] == 'TOPIC':
del h['DIRECTION']
hJson = ujson.dumps(h)
topicHeadersArray.append(hJson)
return {
'PositiveHeaders': positiveHeadersArray,
'NeutralHeaders': neutralHeadersArray,
'NegativeHeaders': negativeHeadersArray,
'TopicHeaders': topicHeadersArray
}
# Here i need to get the bucket, produce a valid amp packet with the data given and return that.
# I also convert to JSON here.
def send(reply): # Here I send the amp packet produced.
global totalPacketNumber
print('L->R: Header reply, Package number: ', CPNo, ' N:',
self.connectedNode['NodeId'])
reply['TotalNumberOfPackets'] = totalPacketNumber
reply['CurrentPacketNo'] = currentPacketNumber
self.callRemote(
networkAPI.ReceiveHeaders,
PositiveHeaders=reply['PositiveHeaders'],
NeutralHeaders=reply['NeutralHeaders'],
NegativeHeaders=reply['NegativeHeaders'],
TopicHeaders=reply['TopicHeaders'],
TotalNumberOfPackets=reply['TotalNumberOfPackets'],
CurrentPacketNo=reply['CurrentPacketNo'])
return {}
return send(construct(slicedDbReply))
if len(dbReply) > bucketSize:
bucketise(dbReply[bucketSize:], bucketSize,
CPNo + 1) # next batch.
return process(dbReply[0:bucketSize], CPNo) # current batch.
else:
return process(dbReply[0:bucketSize], CPNo) # current batch.
d.addCallback(bucketise, HEADER_PACKET_COUNT, currentPacketNumber)
d.addErrback(self.abortConnection, 'L<-R', Mercury.getHeaders.__name__)
#return d # bad local return?
return {}
def respondWithHeaders(self, LastSyncTimestamp, Languages):
#Check if handshake actually occurred. If not, abort.
if not self.connectedNode:
self.abortConnection(Exception('AetherError: Illegal request from remote.'), 'L<-R', 'respondWithHeaders')
return {}
if LastSyncTimestamp != 'null':
LastSyncTimestamp = datetime.utcfromtimestamp(float(LastSyncTimestamp))
else:
LastSyncTimestamp = 'NO-TIMESTAMP'
print('L<-R: Headers request, Timestamp: ', LastSyncTimestamp if LastSyncTimestamp != 'NO-TIMESTAMP' else 'Initial Connection', ' N:', self.connectedNode['NodeId'])
d = Mercury.getHeaders(LastSyncTimestamp, Languages)
currentPacketNumber = 1
totalPacketNumber = 1 # start from one.
def calculatePacketCounts(dbReply, bucketSize):
global totalPacketNumber
totalPacketNumber = len(dbReply)/bucketSize
if len(dbReply)%bucketSize > 0:
totalPacketNumber+= 1
if len(dbReply) == 0:
totalPacketNumber = 1
return dbReply
d.addCallback(calculatePacketCounts, globals.headerPacketCount)
def bucketise(dbReply, bucketSize, CPNo):
def process(slicedDbReply, currentPacketNumber):
# Here I need to fire the bucket given.
def construct(bucket):
positiveHeadersArray = []
neutralHeadersArray = []
negativeHeadersArray = []
topicHeadersArray = []
for h in bucket:
if h['DIRECTION'] == 'POSITIVE':
del h['DIRECTION']
hJson = ujson.dumps(h)
positiveHeadersArray.append(hJson)
elif h['DIRECTION'] == 'NEUTRAL':
del h['DIRECTION']
hJson = ujson.dumps(h)
neutralHeadersArray.append(hJson)
elif h['DIRECTION'] == 'NEGATIVE':
del h['DIRECTION']
hJson = ujson.dumps(h)
negativeHeadersArray.append(hJson)
elif h['DIRECTION'] == 'TOPIC':
del h['DIRECTION']
hJson = ujson.dumps(h)
topicHeadersArray.append(hJson)
return {
'PositiveHeaders': positiveHeadersArray,
'NeutralHeaders': neutralHeadersArray,
'NegativeHeaders': negativeHeadersArray,
'TopicHeaders': topicHeadersArray
}
# Here i need to get the bucket, produce a valid amp packet with the data given and return that.
# I also convert to JSON here.
def send(reply): # Here I send the amp packet produced.
global totalPacketNumber
print('L->R: Header reply, Package number: ', CPNo, ' N:', self.connectedNode['NodeId'])
reply['TotalNumberOfPackets'] = totalPacketNumber
reply['CurrentPacketNo'] = currentPacketNumber
self.callRemote(networkAPI.ReceiveHeaders,
PositiveHeaders = reply['PositiveHeaders'],
NeutralHeaders = reply['NeutralHeaders'],
NegativeHeaders = reply['NegativeHeaders'],
TopicHeaders = reply['TopicHeaders'],
TotalNumberOfPackets = reply['TotalNumberOfPackets'],
CurrentPacketNo = reply['CurrentPacketNo']
)
return {}
return send(construct(slicedDbReply))
if len(dbReply) > bucketSize:
bucketise(dbReply[bucketSize:], bucketSize, CPNo + 1) # next batch.
return process(dbReply[0:bucketSize], CPNo) # current batch.
else:
return process(dbReply[0:bucketSize], CPNo) # current batch.
d.addCallback(bucketise, globals.headerPacketCount, currentPacketNumber)
d.addErrback(self.abortConnection, 'L<-R', Mercury.getHeaders.__name__)
#return d # bad local return?
return {}
def respondWithHeaders(self, LastSyncTimestamp, Languages):
if LastSyncTimestamp != 'null':
LastSyncTimestamp = datetime.utcfromtimestamp(
float(LastSyncTimestamp)) # + timedelta(minutes = 1)
else:
LastSyncTimestamp = 'NO-TIMESTAMP'
cprint('FROM REMOTE: OLD LOCAL TIMESTAMP: %s' %
(LastSyncTimestamp
if LastSyncTimestamp != 'NO-TIMESTAMP' else LastSyncTimestamp),
'white',
'on_yellow',
attrs=['bold'])
print('\n')
d = Mercury.getHeaders(LastSyncTimestamp, Languages)
currentPacketNumber = 1
totalPacketNumber = 1 # start from one.
def calculatePacketCounts(dbReply, bucketSize):
global totalPacketNumber
totalPacketNumber = len(dbReply) / bucketSize
if len(dbReply) % bucketSize > 0:
totalPacketNumber += 1
if len(dbReply) == 0:
totalPacketNumber = 1
print('the reply includes %s headers' % len(dbReply))
print('total packet numbers: %s' % totalPacketNumber)
return dbReply
d.addCallback(calculatePacketCounts, globals.headerPacketCount)
def bucketise(dbReply, bucketSize, CPNo):
def process(slicedDbReply, currentPacketNumber):
# here I need to fire the bucket given.
def construct(bucket):
positiveHeadersArray = []
neutralHeadersArray = []
negativeHeadersArray = []
topicHeadersArray = []
for h in bucket:
if h['DIRECTION'] == 'POSITIVE':
del h['DIRECTION']
hJson = ujson.dumps(h)
positiveHeadersArray.append(hJson)
elif h['DIRECTION'] == 'NEUTRAL':
del h['DIRECTION']
hJson = ujson.dumps(h)
neutralHeadersArray.append(hJson)
elif h['DIRECTION'] == 'NEGATIVE':
del h['DIRECTION']
hJson = ujson.dumps(h)
negativeHeadersArray.append(hJson)
elif h['DIRECTION'] == 'TOPIC':
del h['DIRECTION']
hJson = ujson.dumps(h)
topicHeadersArray.append(hJson)
return {
'PositiveHeaders': positiveHeadersArray,
'NeutralHeaders': neutralHeadersArray,
'NegativeHeaders': negativeHeadersArray,
'TopicHeaders': topicHeadersArray
}
# Here i need to get the bucket, produce a valid amp packet with the data given and return that.
# I ALSO CONVERT TO JSON HERE.
def send(reply): # here I send the amp packet produced.
global totalPacketNumber
cprint('THIS IS PACKAGE NUMBERED: %d' % CPNo, 'white',
'on_red')
reply['TotalNumberOfPackets'] = totalPacketNumber
reply['CurrentPacketNo'] = currentPacketNumber
print('reply: ', reply)
# here I need to fire receiveheaders methods onto the remote for each of the packets.
self.callRemote(
networkAPI.ReceiveHeaders,
PositiveHeaders=reply['PositiveHeaders'],
NeutralHeaders=reply['NeutralHeaders'],
NegativeHeaders=reply['NegativeHeaders'],
TopicHeaders=reply['TopicHeaders'],
TotalNumberOfPackets=reply['TotalNumberOfPackets'],
CurrentPacketNo=reply['CurrentPacketNo'])
return {}
return send(construct(slicedDbReply))
if len(dbReply) > bucketSize:
bucketise(dbReply[bucketSize:], bucketSize,
CPNo + 1) # next batch.
return process(dbReply[0:bucketSize], CPNo) # current batch.
else:
return process(dbReply[0:bucketSize], CPNo) # current batch.
d.addCallback(bucketise, globals.headerPacketCount,
currentPacketNumber)
d.addErrback(self.closeConnection, 'INBOUND',
Mercury.getHeaders.__name__)
return d
def processNode(nodeExists):
if nodeExists:
return Mercury.updateAndGetNode(
NodeId, ip, ListeningPort) # This returns deferred
else:
return Mercury.createNode(NodeId, ip, ListeningPort)
def replyArrived(reply):
return Mercury.insertPost(reply, self.ephemeralConnectionId)
def respondWithHeaders(self, LastSyncTimestamp, Languages):
# Check if handshake actually occurred. If not, abort.
if not self.connectedNode:
self.abortConnection(Exception("AetherError: Illegal request from remote."), "L<-R", "respondWithHeaders")
return {}
if LastSyncTimestamp != "null":
LastSyncTimestamp = datetime.utcfromtimestamp(float(LastSyncTimestamp))
else:
LastSyncTimestamp = "NO-TIMESTAMP"
print(
"L<-R: Headers request, Timestamp: ",
LastSyncTimestamp if LastSyncTimestamp != "NO-TIMESTAMP" else "Initial Connection",
" N:",
self.connectedNode["NodeId"],
)
d = Mercury.getHeaders(LastSyncTimestamp, Languages)
currentPacketNumber = 1
totalPacketNumber = 1 # start from one.
def calculatePacketCounts(dbReply, bucketSize):
global totalPacketNumber
totalPacketNumber = len(dbReply) / bucketSize
if len(dbReply) % bucketSize > 0:
totalPacketNumber += 1
if len(dbReply) == 0:
totalPacketNumber = 1
return dbReply
d.addCallback(calculatePacketCounts, HEADER_PACKET_COUNT)
def bucketise(dbReply, bucketSize, CPNo):
def process(slicedDbReply, currentPacketNumber):
# Here I need to fire the bucket given.
def construct(bucket):
positiveHeadersArray = []
neutralHeadersArray = []
negativeHeadersArray = []
topicHeadersArray = []
for h in bucket:
if h["DIRECTION"] == "POSITIVE":
del h["DIRECTION"]
hJson = ujson.dumps(h)
positiveHeadersArray.append(hJson)
elif h["DIRECTION"] == "NEUTRAL":
del h["DIRECTION"]
hJson = ujson.dumps(h)
neutralHeadersArray.append(hJson)
elif h["DIRECTION"] == "NEGATIVE":
del h["DIRECTION"]
hJson = ujson.dumps(h)
negativeHeadersArray.append(hJson)
elif h["DIRECTION"] == "TOPIC":
del h["DIRECTION"]
hJson = ujson.dumps(h)
topicHeadersArray.append(hJson)
return {
"PositiveHeaders": positiveHeadersArray,
"NeutralHeaders": neutralHeadersArray,
"NegativeHeaders": negativeHeadersArray,
"TopicHeaders": topicHeadersArray,
}
# Here i need to get the bucket, produce a valid amp packet with the data given and return that.
# I also convert to JSON here.
def send(reply): # Here I send the amp packet produced.
global totalPacketNumber
print("L->R: Header reply, Package number: ", CPNo, " N:", self.connectedNode["NodeId"])
reply["TotalNumberOfPackets"] = totalPacketNumber
reply["CurrentPacketNo"] = currentPacketNumber
self.callRemote(
networkAPI.ReceiveHeaders,
PositiveHeaders=reply["PositiveHeaders"],
NeutralHeaders=reply["NeutralHeaders"],
NegativeHeaders=reply["NegativeHeaders"],
TopicHeaders=reply["TopicHeaders"],
TotalNumberOfPackets=reply["TotalNumberOfPackets"],
CurrentPacketNo=reply["CurrentPacketNo"],
)
return {}
return send(construct(slicedDbReply))
if len(dbReply) > bucketSize:
bucketise(dbReply[bucketSize:], bucketSize, CPNo + 1) # next batch.
return process(dbReply[0:bucketSize], CPNo) # current batch.
else:
return process(dbReply[0:bucketSize], CPNo) # current batch.
d.addCallback(bucketise, HEADER_PACKET_COUNT, currentPacketNumber)
d.addErrback(self.abortConnection, "L<-R", Mercury.getHeaders.__name__)
# return d # bad local return?
return {}
def receiveNewSyncTimestamp(self, NewSyncTimestamp):
newSyncTimestamp = NewSyncTimestamp
cprint('FROM REMOTE: NEW REMOTE TIMESTAMP: %s' %newSyncTimestamp, 'white', 'on_yellow', attrs=['bold'])
print('\n')
Mercury.insertNewSyncTimestamp(self.connectedNode, newSyncTimestamp)
return {}
def replyArrived(reply):
return Mercury.insertPost(reply, self.ephemeralConnectionId)
def respondWithNodes(self, LastSyncTimestamp):
if LastSyncTimestamp != 'null':
LastSyncTimestamp = datetime.utcfromtimestamp(float(LastSyncTimestamp))# + timedelta(minutes = 1)
else:
LastSyncTimestamp = 'NO-TIMESTAMP'
d = Mercury.getNodes(LastSyncTimestamp)
d.addCallback(Mercury.processNodes)
cprint('FROM REMOTE: NODES REQUEST: with sync timestamp %s' %(LastSyncTimestamp
if LastSyncTimestamp != 'NO-TIMESTAMP' else LastSyncTimestamp), 'white', 'on_yellow', attrs=['bold'])
print('\n')
currentPacketNumber = 1
totalPacketNumber = 1
def calculatePacketCounts(dbReply, bucketSize):
global totalPacketNumber
totalPacketNumber = len(dbReply)/bucketSize
if len(dbReply)%bucketSize > 0:
totalPacketNumber+= 1
if len(dbReply) == 0:
totalPacketNumber = 1
print('the reply includes %s headers' %len(dbReply))
print('total packet numbers: %s' %totalPacketNumber)
return dbReply
d.addCallback(calculatePacketCounts, globals.nodePacketCount)
def bucketise(dbReply, bucketSize ,CPNo):
# Here I have the entire reply from the db. I need divide that reply into buckets, and fire each
# bucket to the remote.
def process(slicedDbReply, currentPacketNumber):
global totalPacketNumber
nodesJson = []
for n in slicedDbReply:
nJson = ujson.dumps(n)
nodesJson.append(nJson)
cprint('THIS IS NODE ANSWER PACKAGE NUMBERED: %d' %currentPacketNumber, 'white', 'on_red')
print('reply: ',{'Nodes':nodesJson,
'TotalNumberOfPackets':totalPacketNumber,
'CurrentPacketNo':currentPacketNumber})
# here I need to fire receiveheaders methods onto the remote for each of the packets.
self.callRemote(networkAPI.ReceiveNodes,
Nodes = nodesJson,
TotalNumberOfPackets = totalPacketNumber,
CurrentPacketNo = currentPacketNumber
)
return {}
if len(dbReply) > bucketSize:
bucketise(dbReply[bucketSize:], bucketSize, CPNo + 1) # next batch.
return process(dbReply[0:bucketSize], CPNo) # current batch.
else:
return process(dbReply[0:bucketSize], CPNo) # current batch.
d.addCallback(bucketise, globals.nodePacketCount, currentPacketNumber)
d.addErrback(self.closeConnection, 'INBOUND', self.respondWithNodes.__name__)
return d
def processNode(nodeExists):
if nodeExists:
return Mercury.updateAndGetNode(NodeId, ip, ListeningPort) # This returns deferred
else:
return Mercury.createNode(NodeId, ip, ListeningPort)
def respondWithHeaders(self, LastSyncTimestamp, Languages):
if LastSyncTimestamp != 'null':
LastSyncTimestamp = datetime.utcfromtimestamp(float(LastSyncTimestamp))# + timedelta(minutes = 1)
else:
LastSyncTimestamp = 'NO-TIMESTAMP'
cprint('FROM REMOTE: OLD LOCAL TIMESTAMP: %s' %(LastSyncTimestamp
if LastSyncTimestamp != 'NO-TIMESTAMP' else LastSyncTimestamp), 'white', 'on_yellow', attrs=['bold'])
print('\n')
d = Mercury.getHeaders(LastSyncTimestamp, Languages)
currentPacketNumber = 1
totalPacketNumber = 1 # start from one.
def calculatePacketCounts(dbReply, bucketSize):
global totalPacketNumber
totalPacketNumber = len(dbReply)/bucketSize
if len(dbReply)%bucketSize > 0:
totalPacketNumber+= 1
if len(dbReply) == 0:
totalPacketNumber = 1
print('the reply includes %s headers' %len(dbReply))
print('total packet numbers: %s' %totalPacketNumber)
return dbReply
d.addCallback(calculatePacketCounts, globals.headerPacketCount)
def bucketise(dbReply, bucketSize, CPNo):
def process(slicedDbReply, currentPacketNumber):
# here I need to fire the bucket given.
def construct(bucket):
positiveHeadersArray = []
neutralHeadersArray = []
negativeHeadersArray = []
topicHeadersArray = []
for h in bucket:
if h['DIRECTION'] == 'POSITIVE':
del h['DIRECTION']
hJson = ujson.dumps(h)
positiveHeadersArray.append(hJson)
elif h['DIRECTION'] == 'NEUTRAL':
del h['DIRECTION']
hJson = ujson.dumps(h)
neutralHeadersArray.append(hJson)
elif h['DIRECTION'] == 'NEGATIVE':
del h['DIRECTION']
hJson = ujson.dumps(h)
negativeHeadersArray.append(hJson)
elif h['DIRECTION'] == 'TOPIC':
del h['DIRECTION']
hJson = ujson.dumps(h)
topicHeadersArray.append(hJson)
return {
'PositiveHeaders': positiveHeadersArray,
'NeutralHeaders': neutralHeadersArray,
'NegativeHeaders': negativeHeadersArray,
'TopicHeaders': topicHeadersArray
}
# Here i need to get the bucket, produce a valid amp packet with the data given and return that.
# I ALSO CONVERT TO JSON HERE.
def send(reply): # here I send the amp packet produced.
global totalPacketNumber
cprint('THIS IS PACKAGE NUMBERED: %d' %CPNo, 'white', 'on_red')
reply['TotalNumberOfPackets'] = totalPacketNumber
reply['CurrentPacketNo'] = currentPacketNumber
print('reply: ',reply)
# here I need to fire receiveheaders methods onto the remote for each of the packets.
self.callRemote(networkAPI.ReceiveHeaders,
PositiveHeaders = reply['PositiveHeaders'],
NeutralHeaders = reply['NeutralHeaders'],
NegativeHeaders = reply['NegativeHeaders'],
TopicHeaders = reply['TopicHeaders'],
TotalNumberOfPackets = reply['TotalNumberOfPackets'],
CurrentPacketNo = reply['CurrentPacketNo']
)
return {}
return send(construct(slicedDbReply))
if len(dbReply) > bucketSize:
bucketise(dbReply[bucketSize:], bucketSize, CPNo + 1) # next batch.
return process(dbReply[0:bucketSize], CPNo) # current batch.
else:
return process(dbReply[0:bucketSize], CPNo) # current batch.
d.addCallback(bucketise, globals.headerPacketCount, currentPacketNumber)
d.addErrback(self.closeConnection, 'INBOUND', Mercury.getHeaders.__name__)
return d
