def recinv(self, data): totalNumberOfobjectsThatWeHaveYetToGetFromAllPeers = 0 # this counts duplicates seperately because they take up memory if len(shared.numberOfObjectsThatWeHaveYetToGetPerPeer) > 0: for key, value in shared.numberOfObjectsThatWeHaveYetToGetPerPeer.items(): totalNumberOfobjectsThatWeHaveYetToGetFromAllPeers += value with shared.printLock: print 'number of keys(hosts) in shared.numberOfObjectsThatWeHaveYetToGetPerPeer:', len(shared.numberOfObjectsThatWeHaveYetToGetPerPeer) print 'totalNumberOfobjectsThatWeHaveYetToGetFromAllPeers = ', totalNumberOfobjectsThatWeHaveYetToGetFromAllPeers numberOfItemsInInv, lengthOfVarint = decodeVarint(data[:10]) if numberOfItemsInInv > 50000: sys.stderr.write('Too many items in inv message!') return if len(data) < lengthOfVarint + (numberOfItemsInInv * 32): print 'inv message doesn\'t contain enough data. Ignoring.' return if numberOfItemsInInv == 1: # we'll just request this data from the person who advertised the object. if totalNumberOfobjectsThatWeHaveYetToGetFromAllPeers > 200000 and len(self.objectsThatWeHaveYetToGetFromThisPeer) > 1000: # inv flooding attack mitigation with shared.printLock: print 'We already have', totalNumberOfobjectsThatWeHaveYetToGetFromAllPeers, 'items yet to retrieve from peers and over 1000 from this node in particular. Ignoring this inv message.' return self.someObjectsOfWhichThisRemoteNodeIsAlreadyAware[ data[lengthOfVarint:32 + lengthOfVarint]] = 0 shared.numberOfInventoryLookupsPerformed += 1 if data[lengthOfVarint:32 + lengthOfVarint] in shared.inventory: with shared.printLock: print 'Inventory (in memory) has inventory item already.' elif shared.isInSqlInventory(data[lengthOfVarint:32 + lengthOfVarint]): print 'Inventory (SQL on disk) has inventory item already.' else: self.sendgetdata(data[lengthOfVarint:32 + lengthOfVarint]) else: # There are many items listed in this inv message. Let us create a # 'set' of objects we are aware of and a set of objects in this inv # message so that we can diff one from the other cheaply. startTime = time.time() advertisedSet = set() for i in range(numberOfItemsInInv): advertisedSet.add(data[lengthOfVarint + (32 * i):32 + lengthOfVarint + (32 * i)]) objectsNewToMe = advertisedSet - shared.inventorySets[self.streamNumber] #logger.info('inv message lists %s objects. Of those %s are new to me. It took %s seconds to figure that out.', numberOfItemsInInv, len(objectsNewToMe), time.time()-startTime) for item in objectsNewToMe: if totalNumberOfobjectsThatWeHaveYetToGetFromAllPeers > 200000 and len(self.objectsThatWeHaveYetToGetFromThisPeer) > 1000: # inv flooding attack mitigation with shared.printLock: print 'We already have', totalNumberOfobjectsThatWeHaveYetToGetFromAllPeers, 'items yet to retrieve from peers and over', len(self.objectsThatWeHaveYetToGetFromThisPeer), 'from this node in particular. Ignoring the rest of this inv message.' break self.someObjectsOfWhichThisRemoteNodeIsAlreadyAware[item] = 0 # helps us keep from sending inv messages to peers that already know about the objects listed therein self.objectsThatWeHaveYetToGetFromThisPeer[item] = 0 # upon finishing dealing with an incoming message, the receiveDataThread will request a random object of from peer out of this data structure. This way if we get multiple inv messages from multiple peers which list mostly the same objects, we will make getdata requests for different random objects from the various peers. if len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0: shared.numberOfObjectsThatWeHaveYetToGetPerPeer[ self.peer] = len(self.objectsThatWeHaveYetToGetFromThisPeer)
def processData(self): # if shared.verbose >= 3: # with shared.printLock: # print 'self.data is currently ', repr(self.data) # if ( len(self.data) < shared.Header.size ): # if so little of the data has arrived that we can't even read the checksum then wait for more data. return # Use a memoryview so we don't copy data unnecessarily view = memoryview(self.data) magic, command, payloadLength, checksum = shared.Header.unpack(view[: shared.Header.size]) view = view[shared.Header.size :] if magic != 0xE9BEB4D9: # if shared.verbose >= 1: # with shared.printLock: # print 'The magic bytes were not correct. First 40 bytes of data: ' + repr(self.data[0:40]) self.data = "" return if payloadLength > 20000000: logger.info( "The incoming message, which we have not yet download, is too large. Ignoring it. (unfortunately there is no way to tell the other node to stop sending it except to disconnect.) Message size: %s" % payloadLength ) self.data = view[payloadLength:].tobytes() del view, magic, command, payloadLength, checksum # we don't need these anymore and better to clean them now before the recursive call rather than after self.processData() return if len(view) < payloadLength: # check if the whole message has arrived yet. return payload = view[:payloadLength] if checksum != hashlib.sha512(payload).digest()[0:4]: # test the checksum in the message. print "Checksum incorrect. Clearing this message." self.data = view[payloadLength:].tobytes() del view, magic, command, payloadLength, checksum, payload # again better to clean up before the recursive call self.processData() return # We can now revert back to bytestrings and take this message out payload = payload.tobytes() self.data = view[payloadLength:].tobytes() del view, magic, payloadLength, checksum # The time we've last seen this node is obviously right now since we # just received valid data from it. So update the knownNodes list so # that other peers can be made aware of its existance. if ( self.initiatedConnection and self.connectionIsOrWasFullyEstablished ): # The remote port is only something we should share with others if it is the remote node's incoming port (rather than some random operating-system-assigned outgoing port). shared.knownNodesLock.acquire() shared.knownNodes[self.streamNumber][self.peer] = int(time.time()) shared.knownNodesLock.release() # Strip the nulls command = command.rstrip("\x00") with shared.printLock: print "remoteCommand", repr(command), " from", self.peer # TODO: Use a dispatcher here if not self.connectionIsOrWasFullyEstablished: if command == "version": self.recversion(payload) elif command == "verack": self.recverack() else: if command == "addr": self.recaddr(payload) elif command == "getpubkey": shared.checkAndSharegetpubkeyWithPeers(payload) elif command == "pubkey": self.recpubkey(payload) elif command == "inv": self.recinv(payload) elif command == "getdata": self.recgetdata(payload) elif command == "msg": self.recmsg(payload) elif command == "broadcast": self.recbroadcast(payload) elif command == "ping": self.sendpong(payload) # elif command == 'pong': # pass # elif command == 'alert': # pass if self.data == "": while len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0: shared.numberOfInventoryLookupsPerformed += 1 objectHash, = random.sample(self.objectsThatWeHaveYetToGetFromThisPeer, 1) if objectHash in shared.inventory: with shared.printLock: print "Inventory (in memory) already has object listed in inv message." del self.objectsThatWeHaveYetToGetFromThisPeer[objectHash] elif shared.isInSqlInventory(objectHash): if shared.verbose >= 3: with shared.printLock: print "Inventory (SQL on disk) already has object listed in inv message." del self.objectsThatWeHaveYetToGetFromThisPeer[objectHash] else: self.sendgetdata(objectHash) del self.objectsThatWeHaveYetToGetFromThisPeer[ objectHash ] # It is possible that the remote node might not respond with the object. In that case, we'll very likely get it from someone else anyway. if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0: with shared.printLock: print "(concerning", str( self.peer ) + ")", "number of objectsThatWeHaveYetToGetFromThisPeer is now", len( self.objectsThatWeHaveYetToGetFromThisPeer ) try: del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[ self.peer ] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together. except: pass break if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0: with shared.printLock: print "(concerning", str( self.peer ) + ")", "number of objectsThatWeHaveYetToGetFromThisPeer is now", len( self.objectsThatWeHaveYetToGetFromThisPeer ) try: del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[ self.peer ] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together. except: pass if len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0: with shared.printLock: print "(concerning", str( self.peer ) + ")", "number of objectsThatWeHaveYetToGetFromThisPeer is now", len( self.objectsThatWeHaveYetToGetFromThisPeer ) shared.numberOfObjectsThatWeHaveYetToGetPerPeer[self.peer] = len( self.objectsThatWeHaveYetToGetFromThisPeer ) # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together. self.processData()
def processData(self): # if shared.verbose >= 3: # with shared.printLock: # print 'self.data is currently ', repr(self.data) # if len(self.data) < 24: # if so little of the data has arrived that we can't even read the checksum then wait for more data. return if self.data[0:4] != '\xe9\xbe\xb4\xd9': if shared.verbose >= 1: with shared.printLock: print 'The magic bytes were not correct. First 40 bytes of data: ' + repr(self.data[0:40]) self.data = "" return self.payloadLength, = unpack('>L', self.data[16:20]) if len(self.data) < self.payloadLength + 24: # check if the whole message has arrived yet. return if self.data[20:24] != hashlib.sha512(self.data[24:self.payloadLength + 24]).digest()[0:4]: # test the checksum in the message. If it is correct... print 'Checksum incorrect. Clearing this message.' self.data = self.data[self.payloadLength + 24:] self.processData() return # The time we've last seen this node is obviously right now since we # just received valid data from it. So update the knownNodes list so # that other peers can be made aware of its existance. if self.initiatedConnection and self.connectionIsOrWasFullyEstablished: # The remote port is only something we should share with others if it is the remote node's incoming port (rather than some random operating-system-assigned outgoing port). shared.knownNodesLock.acquire() shared.knownNodes[self.streamNumber][self.peer] = int(time.time()) shared.knownNodesLock.release() if self.payloadLength <= 180000000: # If the size of the message is greater than 180MB, ignore it. (I get memory errors when processing messages much larger than this though it is concievable that this value will have to be lowered if some systems are less tolarant of large messages.) remoteCommand = self.data[4:16] with shared.printLock: print 'remoteCommand', repr(remoteCommand.replace('\x00', '')), ' from', self.peer if remoteCommand == 'version\x00\x00\x00\x00\x00': self.recversion(self.data[24:self.payloadLength + 24]) elif remoteCommand == 'verack\x00\x00\x00\x00\x00\x00': self.recverack() elif remoteCommand == 'addr\x00\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished: self.recaddr(self.data[24:self.payloadLength + 24]) elif remoteCommand == 'getpubkey\x00\x00\x00' and self.connectionIsOrWasFullyEstablished: shared.checkAndSharegetpubkeyWithPeers(self.data[24:self.payloadLength + 24]) elif remoteCommand == 'pubkey\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished: self.recpubkey(self.data[24:self.payloadLength + 24]) elif remoteCommand == 'inv\x00\x00\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished: self.recinv(self.data[24:self.payloadLength + 24]) elif remoteCommand == 'getdata\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished: self.recgetdata(self.data[24:self.payloadLength + 24]) elif remoteCommand == 'msg\x00\x00\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished: self.recmsg(self.data[24:self.payloadLength + 24]) elif remoteCommand == 'broadcast\x00\x00\x00' and self.connectionIsOrWasFullyEstablished: self.recbroadcast(self.data[24:self.payloadLength + 24]) elif remoteCommand == 'ping\x00\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished: self.sendpong() elif remoteCommand == 'pong\x00\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished: pass elif remoteCommand == 'alert\x00\x00\x00\x00\x00\x00\x00' and self.connectionIsOrWasFullyEstablished: pass self.data = self.data[ self.payloadLength + 24:] # take this message out and then process the next message if self.data == '': while len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0: shared.numberOfInventoryLookupsPerformed += 1 objectHash, = random.sample( self.objectsThatWeHaveYetToGetFromThisPeer, 1) if objectHash in shared.inventory: with shared.printLock: print 'Inventory (in memory) already has object listed in inv message.' del self.objectsThatWeHaveYetToGetFromThisPeer[ objectHash] elif shared.isInSqlInventory(objectHash): if shared.verbose >= 3: with shared.printLock: print 'Inventory (SQL on disk) already has object listed in inv message.' del self.objectsThatWeHaveYetToGetFromThisPeer[ objectHash] else: self.sendgetdata(objectHash) del self.objectsThatWeHaveYetToGetFromThisPeer[ objectHash] # It is possible that the remote node might not respond with the object. In that case, we'll very likely get it from someone else anyway. if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0: with shared.printLock: print '(concerning', str(self.peer) + ')', 'number of objectsThatWeHaveYetToGetFromThisPeer is now', len(self.objectsThatWeHaveYetToGetFromThisPeer) try: del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[ self.peer] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together. except: pass break if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0: with shared.printLock: print '(concerning', str(self.peer) + ')', 'number of objectsThatWeHaveYetToGetFromThisPeer is now', len(self.objectsThatWeHaveYetToGetFromThisPeer) try: del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[ self.peer] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together. except: pass if len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0: with shared.printLock: print '(concerning', str(self.peer) + ')', 'number of objectsThatWeHaveYetToGetFromThisPeer is now', len(self.objectsThatWeHaveYetToGetFromThisPeer) shared.numberOfObjectsThatWeHaveYetToGetPerPeer[self.peer] = len( self.objectsThatWeHaveYetToGetFromThisPeer) # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together. self.processData()
def processData(self): if len(self.data) < shared.Header.size: # if so little of the data has arrived that we can't even read the checksum then wait for more data. return magic,command,payloadLength,checksum = shared.Header.unpack(self.data[:shared.Header.size]) if magic != 0xE9BEB4D9: self.data = "" return if payloadLength > 1600100: # ~1.6 MB which is the maximum possible size of an inv message. logger.info('The incoming message, which we have not yet download, is too large. Ignoring it. (unfortunately there is no way to tell the other node to stop sending it except to disconnect.) Message size: %s' % payloadLength) self.data = self.data[payloadLength + shared.Header.size:] del magic,command,payloadLength,checksum # we don't need these anymore and better to clean them now before the recursive call rather than after self.processData() return if len(self.data) < payloadLength + shared.Header.size: # check if the whole message has arrived yet. return payload = self.data[shared.Header.size:payloadLength + shared.Header.size] if checksum != hashlib.sha512(payload).digest()[0:4]: # test the checksum in the message. print 'Checksum incorrect. Clearing this message.' self.data = self.data[payloadLength + shared.Header.size:] del magic,command,payloadLength,checksum,payload # better to clean up before the recursive call self.processData() return # The time we've last seen this node is obviously right now since we # just received valid data from it. So update the knownNodes list so # that other peers can be made aware of its existance. if self.initiatedConnection and self.connectionIsOrWasFullyEstablished: # The remote port is only something we should share with others if it is the remote node's incoming port (rather than some random operating-system-assigned outgoing port). shared.knownNodesLock.acquire() shared.knownNodes[self.streamNumber][self.peer] = int(time.time()) shared.knownNodesLock.release() #Strip the nulls command = command.rstrip('\x00') with shared.printLock: print 'remoteCommand', repr(command), ' from', self.peer try: #TODO: Use a dispatcher here if command == 'error': self.recerror(payload) elif not self.connectionIsOrWasFullyEstablished: if command == 'version': self.recversion(payload) elif command == 'verack': self.recverack() else: if command == 'addr': self.recaddr(payload) elif command == 'inv': self.recinv(payload) elif command == 'getdata': self.recgetdata(payload) elif command == 'object': self.recobject(payload) elif command == 'ping': self.sendpong(payload) #elif command == 'pong': # pass except varintDecodeError as e: logger.debug("There was a problem with a varint while processing a message from the wire. Some details: %s" % e) except Exception as e: logger.critical("Critical error in a receiveDataThread: \n%s" % traceback.format_exc()) del payload self.data = self.data[payloadLength + shared.Header.size:] # take this message out and then process the next message if self.data == '': # if there are no more messages while len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0: shared.numberOfInventoryLookupsPerformed += 1 objectHash, = random.sample( self.objectsThatWeHaveYetToGetFromThisPeer, 1) if objectHash in shared.inventory: with shared.printLock: print 'Inventory (in memory) already has object listed in inv message.' del self.objectsThatWeHaveYetToGetFromThisPeer[ objectHash] elif shared.isInSqlInventory(objectHash): if shared.verbose >= 3: with shared.printLock: print 'Inventory (SQL on disk) already has object listed in inv message.' del self.objectsThatWeHaveYetToGetFromThisPeer[ objectHash] else: # We don't have the object in our inventory. Let's request it. self.sendgetdata(objectHash) del self.objectsThatWeHaveYetToGetFromThisPeer[ objectHash] # It is possible that the remote node might not respond with the object. In that case, we'll very likely get it from someone else anyway. if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0: with shared.printLock: print '(concerning', str(self.peer) + ')', 'number of objectsThatWeHaveYetToGetFromThisPeer is now 0' try: del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[ self.peer] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together. except: pass break if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0: # We had objectsThatWeHaveYetToGetFromThisPeer but the loop ran, they were all in our inventory, and now we don't have any to get anymore. with shared.printLock: print '(concerning', str(self.peer) + ')', 'number of objectsThatWeHaveYetToGetFromThisPeer is now 0' try: del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[ self.peer] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together. except: pass if len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0: with shared.printLock: print '(concerning', str(self.peer) + ')', 'number of objectsThatWeHaveYetToGetFromThisPeer is now', len(self.objectsThatWeHaveYetToGetFromThisPeer) shared.numberOfObjectsThatWeHaveYetToGetPerPeer[self.peer] = len( self.objectsThatWeHaveYetToGetFromThisPeer) # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together. self.processData()
def __haveInventory__(self, data): if data in shared.inventory: return True return shared.isInSqlInventory(data)
def processData(self): if len(self.data) < shared.Header.size: # if so little of the data has arrived that we can't even read the checksum then wait for more data. return magic,command,payloadLength,checksum = shared.Header.unpack(self.data[:shared.Header.size]) if magic != 0xE9BEB4D9: self.data = "" return if payloadLength > 1600100: # ~1.6 MB which is the maximum possible size of an inv message. self.data = self.data[payloadLength + shared.Header.size:] del magic,command,payloadLength,checksum # we don't need these anymore and better to clean them now before the recursive call rather than after self.processData() return if len(self.data) < payloadLength + shared.Header.size: # check if the whole message has arrived yet. return payload = self.data[shared.Header.size:payloadLength + shared.Header.size] if checksum != hashlib.sha512(payload).digest()[0:4]: # test the checksum in the message. self.data = self.data[payloadLength + shared.Header.size:] del magic,command,payloadLength,checksum,payload # better to clean up before the recursive call self.processData() return # The time we've last seen this node is obviously right now since we # just received valid data from it. So update the knownNodes list so # that other peers can be made aware of its existance. if self.initiatedConnection and self.connectionIsOrWasFullyEstablished: # The remote port is only something we should share with others if it is the remote node's incoming port (rather than some random operating-system-assigned outgoing port). with shared.knownNodesLock: shared.knownNodes[self.streamNumber][self.peer] = int(time.time()) #Strip the nulls command = command.rstrip('\x00') #print "Received %s packet from %s" % (command, str(self.peer)) try: #TODO: Use a dispatcher here if command == 'error': self.recerror(payload) elif not self.connectionIsOrWasFullyEstablished: if command == 'version': self.recversion(payload) elif command == 'verack': self.recverack() else: if command == 'addr': self.recaddr(payload) elif command == 'inv': self.recinv(payload) elif command == 'getdata': self.recgetdata(payload) elif command == 'object': self.recobject(payload) elif command == 'ping': self.sendpong(payload) #elif command == 'pong': # pass except varintDecodeError as e: pass except Exception as e: pass del payload self.data = self.data[payloadLength + shared.Header.size:] # take this message out and then process the next message if self.data == '': # if there are no more messages while len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0: shared.numberOfInventoryLookupsPerformed += 1 objectHash, = random.sample( self.objectsThatWeHaveYetToGetFromThisPeer, 1) if objectHash in shared.inventory: del self.objectsThatWeHaveYetToGetFromThisPeer[ objectHash] elif shared.isInSqlInventory(objectHash): del self.objectsThatWeHaveYetToGetFromThisPeer[ objectHash] else: # We don't have the object in our inventory. Let's request it. self.sendgetdata(objectHash) del self.objectsThatWeHaveYetToGetFromThisPeer[ objectHash] # It is possible that the remote node might not respond with the object. In that case, we'll very likely get it from someone else anyway. if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0: try: del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[ self.peer] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together. except: pass break if len(self.objectsThatWeHaveYetToGetFromThisPeer) == 0: # We had objectsThatWeHaveYetToGetFromThisPeer but the loop ran, they were all in our inventory, and now we don't have any to get anymore. try: del shared.numberOfObjectsThatWeHaveYetToGetPerPeer[ self.peer] # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together. except: pass if len(self.objectsThatWeHaveYetToGetFromThisPeer) > 0: shared.numberOfObjectsThatWeHaveYetToGetPerPeer[self.peer] = len( self.objectsThatWeHaveYetToGetFromThisPeer) # this data structure is maintained so that we can keep track of how many total objects, across all connections, are currently outstanding. If it goes too high it can indicate that we are under attack by multiple nodes working together. self.processData()