def processData(self):
# if shared.verbose >= 3:
# with shared.printLock:
# print 'self.data is currently ', repr(self.data)
#
if len(self.data) < 24: # if so little of the data has arrived that we can't even read the checksum then wait for more data.
return
if self.data[0:4] != '\xe9\xbe\xb4\xd9':
if shared.verbose >= 1:
with shared.printLock:
print 'The magic bytes were not correct. First 40 bytes of data: ' + repr(self.data[0:40])
self.data = ""
return
self.payloadLength, = unpack('>L', self.data[16:20])
if len(self.data) < self.payloadLength + 24: # check if the whole message has arrived yet.
return
if self.data[20:24] != hashlib.sha512(self.data[24:self.payloadLength + 24]).digest()[0:4]: # test the checksum in the message. If it is correct...
print 'Checksum incorrect. Clearing this message.'
self.data = self.data[self.payloadLength + 24:]
self.processData()
return
# The time we've last seen this node is obviously right now since we
# just received valid data from it. So update the knownNodes list so
# that other peers can be made aware of its existance.
if self.initiatedConnection and self.connectionIsOrWasFullyEstablished: # The remote port is only something we should share with others if it is the remote node's incoming port (rather than some random operating-system-assigned outgoing port).
shared.knownNodesLock.acquire()
shared.knownNodes[self.streamNumber][self.peer] = int(time.time())
shared.knownNodesLock.release()
if self.payloadLength <= 180000000: # If the size of the message is greater than 180MB, ignore it. (I get memory errors when processing messages much larger than this though it is concievable that this value will have to be lowered if some systems are less tolarant of large messages.)
remoteCommand = self.data[4:16]
with shared.printLock:
print 'remoteCommand', repr(remoteCommand.replace('\x00', '')), ' from', self.peer
if remoteCommand == 'version\x00\x00\x00\x00\x00':
self.recversion(self.data[24:self.payloadLength + 24])
elif remoteCommand == 'verack\x00\x00\x00\x00\x00\x00':
self.recverack()
elif remoteCommand == 'addr\x00\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
self.recaddr(self.data[24:self.payloadLength + 24])
elif remoteCommand == 'getpubkey\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
shared.checkAndSharegetpubkeyWithPeers(self.data[24:self.payloadLength + 24])
elif remoteCommand == 'pubkey\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
self.recpubkey(self.data[24:self.payloadLength + 24])
elif remoteCommand == 'inv\x00\x00\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
self.recinv(self.data[24:self.payloadLength + 24])
elif remoteCommand == 'getdata\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
self.recgetdata(self.data[24:self.payloadLength + 24])
elif remoteCommand == 'msg\x00\x00\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
self.recmsg(self.data[24:self.payloadLength + 24])
elif remoteCommand == 'broadcast\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
self.recbroadcast(self.data[24:self.payloadLength + 24])
elif remoteCommand == 'ping\x00\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
self.sendpong()
elif remoteCommand == 'pong\x00\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
pass
elif remoteCommand == 'alert\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished:
pass
self.data = self.data[
self.payloadLength + 24:] # take this message out and then process the next message
if self.data == '':
while len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0:
shared.numberOfInventoryLookupsPerformed += 1
objectHash, = random.sample(
self.objectsThatWeHaveYetToGetFromThisPeer, 1)
if objectHash in shared.inventory:
with shared.printLock:
print 'Inventory (in memory) already has object listed in inv message.'
del self.objectsThatWeHaveYetToGetFromThisPeer[
objectHash]
elif shared.isInSqlInventory(objectHash):
if shared.verbose >= 3:
with shared.printLock:
print 'Inventory (SQL on disk) already has object listed in inv message.'
del self.objectsThatWeHaveYetToGetFromThisPeer[
objectHash]
else:
self.sendgetdata(objectHash)
del self.objectsThatWeHaveYetToGetFromThisPeer[
objectHash] # It is possible that the remote node might not respond with the object. In that case, we'll very likely get it from someone else anyway.
if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0:
with shared.printLock:
print '(concerning', str(self.peer) + ')', 'number of objectsThatWeHaveYetToGetFromThisPeer is now', len(self.objectsThatWeHaveYetToGetFromThisPeer)
try:
del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[
self.peer] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
except:
pass
break
if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0:
with shared.printLock:
print '(concerning', str(self.peer) + ')', 'number of objectsThatWeHaveYetToGetFromThisPeer is now', len(self.objectsThatWeHaveYetToGetFromThisPeer)
try:
del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[
self.peer] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
except:
pass
if len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0:
with shared.printLock:
print '(concerning', str(self.peer) + ')', 'number of objectsThatWeHaveYetToGetFromThisPeer is now', len(self.objectsThatWeHaveYetToGetFromThisPeer)
shared.numberOfObjectsThatWeHaveYetToGetPerPeer[self.peer] = len(
self.objectsThatWeHaveYetToGetFromThisPeer) # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
self.processData()
def processData(self):
# if shared.verbose >= 3:
# with shared.printLock:
# print 'self.data is currently ', repr(self.data)
#
if (
len(self.data) < shared.Header.size
): # if so little of the data has arrived that we can't even read the checksum then wait for more data.
return
# Use a memoryview so we don't copy data unnecessarily
view = memoryview(self.data)
magic, command, payloadLength, checksum = shared.Header.unpack(view[: shared.Header.size])
view = view[shared.Header.size :]
if magic != 0xE9BEB4D9:
# if shared.verbose >= 1:
# with shared.printLock:
# print 'The magic bytes were not correct. First 40 bytes of data: ' + repr(self.data[0:40])
self.data = ""
return
if payloadLength > 20000000:
logger.info(
"The incoming message, which we have not yet download, is too large. Ignoring it. (unfortunately there is no way to tell the other node to stop sending it except to disconnect.) Message size: %s"
% payloadLength
)
self.data = view[payloadLength:].tobytes()
del view, magic, command, payloadLength, checksum # we don't need these anymore and better to clean them now before the recursive call rather than after
self.processData()
return
if len(view) < payloadLength: # check if the whole message has arrived yet.
return
payload = view[:payloadLength]
if checksum != hashlib.sha512(payload).digest()[0:4]: # test the checksum in the message.
print "Checksum incorrect. Clearing this message."
self.data = view[payloadLength:].tobytes()
del view, magic, command, payloadLength, checksum, payload # again better to clean up before the recursive call
self.processData()
return
# We can now revert back to bytestrings and take this message out
payload = payload.tobytes()
self.data = view[payloadLength:].tobytes()
del view, magic, payloadLength, checksum
# The time we've last seen this node is obviously right now since we
# just received valid data from it. So update the knownNodes list so
# that other peers can be made aware of its existance.
if (
self.initiatedConnection and self.connectionIsOrWasFullyEstablished
): # The remote port is only something we should share with others if it is the remote node's incoming port (rather than some random operating-system-assigned outgoing port).
shared.knownNodesLock.acquire()
shared.knownNodes[self.streamNumber][self.peer] = int(time.time())
shared.knownNodesLock.release()
# Strip the nulls
command = command.rstrip("\x00")
with shared.printLock:
print "remoteCommand", repr(command), " from", self.peer
# TODO: Use a dispatcher here
if not self.connectionIsOrWasFullyEstablished:
if command == "version":
self.recversion(payload)
elif command == "verack":
self.recverack()
else:
if command == "addr":
self.recaddr(payload)
elif command == "getpubkey":
shared.checkAndSharegetpubkeyWithPeers(payload)
elif command == "pubkey":
self.recpubkey(payload)
elif command == "inv":
self.recinv(payload)
elif command == "getdata":
self.recgetdata(payload)
elif command == "msg":
self.recmsg(payload)
elif command == "broadcast":
self.recbroadcast(payload)
elif command == "ping":
self.sendpong(payload)
# elif command == 'pong':
# pass
# elif command == 'alert':
# pass
if self.data == "":
while len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0:
shared.numberOfInventoryLookupsPerformed += 1
objectHash, = random.sample(self.objectsThatWeHaveYetToGetFromThisPeer, 1)
if objectHash in shared.inventory:
with shared.printLock:
print "Inventory (in memory) already has object listed in inv message."
del self.objectsThatWeHaveYetToGetFromThisPeer[objectHash]
elif shared.isInSqlInventory(objectHash):
if shared.verbose >= 3:
with shared.printLock:
print "Inventory (SQL on disk) already has object listed in inv message."
del self.objectsThatWeHaveYetToGetFromThisPeer[objectHash]
else:
self.sendgetdata(objectHash)
del self.objectsThatWeHaveYetToGetFromThisPeer[
objectHash
] # It is possible that the remote node might not respond with the object. In that case, we'll very likely get it from someone else anyway.
if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0:
with shared.printLock:
print "(concerning", str(
self.peer
) + ")", "number of objectsThatWeHaveYetToGetFromThisPeer is now", len(
self.objectsThatWeHaveYetToGetFromThisPeer
)
try:
del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[
self.peer
] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
except:
pass
break
if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0:
with shared.printLock:
print "(concerning", str(
self.peer
) + ")", "number of objectsThatWeHaveYetToGetFromThisPeer is now", len(
self.objectsThatWeHaveYetToGetFromThisPeer
)
try:
del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[
self.peer
] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
except:
pass
if len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0:
with shared.printLock:
print "(concerning", str(
self.peer
) + ")", "number of objectsThatWeHaveYetToGetFromThisPeer is now", len(
self.objectsThatWeHaveYetToGetFromThisPeer
)
shared.numberOfObjectsThatWeHaveYetToGetPerPeer[self.peer] = len(
self.objectsThatWeHaveYetToGetFromThisPeer
) # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
self.processData()
def processData(self):
if len(self.data) < shared.Header.size: # if so little of the data has arrived that we can't even read the checksum then wait for more data.
return
magic,command,payloadLength,checksum = shared.Header.unpack(self.data[:shared.Header.size])
if magic != 0xE9BEB4D9:
self.data = ""
return
if payloadLength > 20000000:
logger.info('The incoming message, which we have not yet download, is too large. Ignoring it. (unfortunately there is no way to tell the other node to stop sending it except to disconnect.) Message size: %s' % payloadLength)
self.data = self.data[payloadLength + shared.Header.size:]
del magic,command,payloadLength,checksum # we don't need these anymore and better to clean them now before the recursive call rather than after
self.processData()
return
if len(self.data) < payloadLength + shared.Header.size: # check if the whole message has arrived yet.
return
payload = self.data[shared.Header.size:payloadLength + shared.Header.size]
if checksum != hashlib.sha512(payload).digest()[0:4]: # test the checksum in the message.
print 'Checksum incorrect. Clearing this message.'
self.data = self.data[payloadLength + shared.Header.size:]
del magic,command,payloadLength,checksum,payload # better to clean up before the recursive call
self.processData()
return
# The time we've last seen this node is obviously right now since we
# just received valid data from it. So update the knownNodes list so
# that other peers can be made aware of its existance.
if self.initiatedConnection and self.connectionIsOrWasFullyEstablished: # The remote port is only something we should share with others if it is the remote node's incoming port (rather than some random operating-system-assigned outgoing port).
shared.knownNodesLock.acquire()
shared.knownNodes[self.streamNumber][self.peer] = int(time.time())
shared.knownNodesLock.release()
#Strip the nulls
command = command.rstrip('\x00')
with shared.printLock:
print 'remoteCommand', repr(command), ' from', self.peer
#TODO: Use a dispatcher here
if not self.connectionIsOrWasFullyEstablished:
if command == 'version':
self.recversion(payload)
elif command == 'verack':
self.recverack()
else:
if command == 'addr':
self.recaddr(payload)
elif command == 'getpubkey':
shared.checkAndSharegetpubkeyWithPeers(payload)
elif command == 'pubkey':
self.recpubkey(payload)
elif command == 'inv':
self.recinv(payload)
elif command == 'getdata':
self.recgetdata(payload)
elif command == 'msg':
self.recmsg(payload)
elif command == 'broadcast':
self.recbroadcast(payload)
elif command == 'ping':
self.sendpong(payload)
#elif command == 'pong':
# pass
#elif command == 'alert':
# pass
del payload
self.data = self.data[payloadLength + shared.Header.size:] # take this message out and then process the next message
if self.data == '': # if there are no more messages
while len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0:
shared.numberOfInventoryLookupsPerformed += 1
objectHash, = random.sample(
self.objectsThatWeHaveYetToGetFromThisPeer, 1)
if objectHash in shared.inventory:
with shared.printLock:
print 'Inventory (in memory) already has object listed in inv message.'
del self.objectsThatWeHaveYetToGetFromThisPeer[
objectHash]
elif shared.isInSqlInventory(objectHash):
if shared.verbose >= 3:
with shared.printLock:
print 'Inventory (SQL on disk) already has object listed in inv message.'
del self.objectsThatWeHaveYetToGetFromThisPeer[
objectHash]
else:
# We don't have the object in our inventory. Let's request it.
self.sendgetdata(objectHash)
del self.objectsThatWeHaveYetToGetFromThisPeer[
objectHash] # It is possible that the remote node might not respond with the object. In that case, we'll very likely get it from someone else anyway.
if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0:
with shared.printLock:
print '(concerning', str(self.peer) + ')', 'number of objectsThatWeHaveYetToGetFromThisPeer is now 0'
try:
del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[
self.peer] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
except:
pass
break
if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0:
# We had objectsThatWeHaveYetToGetFromThisPeer but the loop ran, they were all in our inventory, and now we don't have any to get anymore.
with shared.printLock:
print '(concerning', str(self.peer) + ')', 'number of objectsThatWeHaveYetToGetFromThisPeer is now 0'
try:
del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[
self.peer] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
except:
pass
if len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0:
with shared.printLock:
print '(concerning', str(self.peer) + ')', 'number of objectsThatWeHaveYetToGetFromThisPeer is now', len(self.objectsThatWeHaveYetToGetFromThisPeer)
shared.numberOfObjectsThatWeHaveYetToGetPerPeer[self.peer] = len(
self.objectsThatWeHaveYetToGetFromThisPeer) # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together.
self.processData()
