def recv_ack(self, m, low_level_peer): key = m.payload.split(BTMessage.MSG_ACK, 1)[1] if key in low_level_peer.unacknowledged: del low_level_peer.unacknowledged[key] else: # This can sometimes happen during simultaneous connect debuglog('btnet', "Received malformed ACK from %s: %s" % (str(low_level_peer), key.encode('hex')))
def accept(self, t, addr, magic, sequence): '''This is called whenever a new connection request is received. Step 2 of three-way handshake. Equivalent to TCP listening -> syn-received transition. ''' # addr should be (IP address, port) since it is from recvfrom() if addr in self.syn_sent: # This is a simultaneous connect, where two nodes are # simultaneously connecting to each other. This is expected to` # occur fairly often, as a result of some UDP hole punching # procedure. We must re-use the existing peer magic values, # otherwise there is a race condition where the two peers end # up ignoring each other because of mismatching magic. peer = self.syn_sent[addr] self.syn_received[(addr, magic)] = peer debuglog('btnet', "Peer %s simultaneous connect" % str(peer)) else: if (addr, magic) in self.syn_received: peer = self.syn_received[(addr, magic)] else: peer = LowLevelPeer(addr[0], addr[1], self.event_loop) self.syn_received[(addr, magic)] = peer debuglog('btnet', "Peer %s in syn-received state" % str(peer)) # TODO: expire stuff in syn_received # There is a DoS potential here: a CONNECT flood, analogous # to a TCP SYN flood. assert len(t.split(BTMessage.MSG_CONNECT, 1)[1]) == 2 * HASH_SIZE payload = peer.connect_nonce[:HASH_SIZE] payload += calculate_ack_payload(t, sequence) peer.send_message(BTMessage.MSG_CONNECT_ACK + payload)
def recv_blockstate(self, data, peer): s = StringIO.StringIO(data.split(BTMessage.MSG_BLOCKSTATE, 1)[1]) sha256 = util.deser_uint256(s) if peer.has_header(sha256) == 'header': peer.inflight[sha256].state.deserialize(s) debuglog('btnet', "New block state for %i: \n" % sha256, peer.inflight[sha256]) self.maybe_download_nodes(peer, sha256)
def send_message(self, t, sequence=None): '''Send the contents of t to the peer. If calling this method from elsewhere, don't use the sequence argument. ''' m = BTMessage(t, self.magic, sequence) debuglog('btnet', "Sent to %s: %s" % (str(self), str(m))) self.event_loop.socket.sendto(m.serialize(), self.addr)
def recv_blockstate(self, data, peer): s = StringIO.StringIO(data.split(BTMessage.MSG_BLOCKSTATE, 1)[1]) hash = util.deser_uint256(s) if peer.has_header(hash) == 'header': peer.inflight[hash].state.deserialize(s) debuglog('btcnet', "New block state for %i: \n" % hash, peer.inflight[hash])
def addnode(self, low_level_peer, magic): if not low_level_peer.addr in self.peers: peer = BTPeer(low_level_peer, magic) self.peers[low_level_peer.addr] = peer self.magic_map[(magic, low_level_peer.addr)] = peer debuglog('btnet', "Adding peer %s" % str(peer)) else: debuglog('btnet', "Peer %s:%i already exists" % low_level_peer.addr)
def connect(self, addr): '''Initiate connection to another peer. Step 1 of three-way handshake. Equivalent to TCP closed -> syn-sent transition. ''' peer = LowLevelPeer(addr[0], addr[1], self.event_loop) peer.send_message_acknowledged(BTMessage.MSG_CONNECT + peer.connect_nonce) self.syn_sent[peer.addr] = peer debuglog('btnet', "Connecting to %s" % str(peer))
def recv_node_request(self, data, peer): s = StringIO.StringIO(data.split(BTMessage.MSG_REQUEST_NODES)[1]) sha256 = util.deser_uint256(s) level = ord(s.read(1)) index = util.deser_varint(s) generations = ord(s.read(1)) flags = util.deser_varint(s) debuglog('btnet', "peer %s wants h=%s l=%i i=%i g=%i f=%i" % (str(peer), util.ser_uint256(sha256)[::-1].encode('hex'), level, index, generations, flags)) # fixme: maybe add choke/throttle checks here? self.send_nodes(peer, sha256, level, index, generations, flags)
def remnode(self, peer): if peer: addr = peer.low_level_peer.addr if addr in self.peers: debuglog('btnet', "Removing peer %s" % (str(peer))) del self.peers[addr] del self.magic_map[(peer.incoming_magic, addr)] peer.send_message(BTMessage.MSG_DISCONNECT) peer.close() else: debuglog('btnet', "Peer %s:%i doesn't exist" % addr)
def handle_close(self): for peer in self.peers.values(): self.remnode(peer) if self.event_loop.state != "closed": debuglog('btnet', "close") self.event_loop.state = "closed" try: time.sleep(1) # wait for MSG_DISCONNECT to be sent self.event_loop.socket.close() except: pass
def run(self, udp_listen): '''Begin event loop. The thread context from which this is called will be henceforth known as the event loop thread. This method will only exit when the event loop is stopped. ''' while not self.e_stop.isSet(): self.state = "starting" debuglog('btnet', "starting BT server on %i" % udp_listen) self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.socket.bind(('localhost', udp_listen)) self.event_loop_thread = thread.get_ident() self.state = "running" while self.state != "closed": if self.e_stop.isSet(): break select_timeout = 30 # Check for callbacks current_time = time.time() while self.callback_queue.qsize() > 0: item = self.callback_queue.get(False) if item[0] <= current_time: # Callback delay has passed, so remove item from queue # and do the callback. item[1](*item[2], **item[3]) else: # Still need to wait for delay to pass. Adjust select # timeout to perform the required delay. select_timeout = min(select_timeout, item[0] - current_time) # Don't remove items from queue until delay has passed. self.callback_queue.put(item) break read_list = [self.socket, self.waker.out_end] # select is the only multiplexer available on all # platforms (Linux/BSD/Windows). rd, wr, ex = select.select(read_list, [], [self.socket], select_timeout) for s in rd: if s == self.waker.out_end: self.waker.handle_read() else: self.read_handler() for s in ex: self.close_handler() self.close_handler() if not self.e_stop.isSet(): time.sleep(5) debuglog('btnet', "reconnect")
def recv_node_request(self, data, peer): s = StringIO.StringIO(data.split(BTMessage.MSG_REQUEST_NODES)[1]) sha256 = util.deser_uint256(s) level = ord(s.read(1)) index = util.deser_varint(s) generations = ord(s.read(1)) flags = util.deser_varint(s) debuglog( 'btnet', "peer %s wants h=%s l=%i i=%i g=%i f=%i" % (str(peer), util.ser_uint256(sha256)[::-1].encode('hex'), level, index, generations, flags)) # fixme: maybe add choke/throttle checks here? self.send_nodes(peer, sha256, level, index, generations, flags)
def recv_header(self, data, peer): blk = mininode.CBlock() f = StringIO.StringIO(data.split(BTMessage.MSG_HEADER, 1)[1]) mininode.CBlockHeader.deserialize(blk, f) blk.calc_sha256() self.add_header(blk) if not peer.has_header(blk.sha256): debuglog('btnet', "Received header from %s: %s" % (peer, repr(blk))) else: debuglog('btnet', "Received duplicate header from %s: %s" % (peer, hex(blk.sha256)[2:])) peer.log_header(blk.sha256) self.broadcast_header(blk) self.req_txcount_proof(peer, blk.sha256)
def accept_finish(self, t, addr, magic): '''Step 3 of three-way handshake (final ACK). Equivalent to TCP syn-received -> established transition. ''' # addr should be (IP address, port) since it is from recvfrom() if (addr, magic) in self.syn_received: peer = self.syn_received[(addr, magic)] h = t.split(BTMessage.MSG_ACK, 1)[1] if h == calculate_hash(peer.connect_nonce[:HASH_SIZE]): del self.syn_received[(addr, magic)] self.peer_adder.addnode(peer, magic) else: debuglog('btnet', "Got malformed ACK from %s:%i" % addr) else: debuglog('btnet', "Got unexpected ACK from %s:%i" % addr)
def recv_header(self, data, peer): blk = mininode.CBlock() f = StringIO.StringIO(data.split(BTMessage.MSG_HEADER, 1)[1]) mininode.CBlockHeader.deserialize(blk, f) blk.calc_sha256() self.add_header(blk) if not peer.has_header(blk.sha256): debuglog('btcnet', "Received header from %s: %s" % (peer, repr(blk))) else: debuglog( 'btcnet', "Received duplicate header from %s: %s" % (peer, hex(blk.sha256)[2:])) peer.log_header(blk.sha256) self.broadcast_header(blk)
def recv_nodes(self, data, peer): s = StringIO.StringIO(data.split(BTMessage.MSG_RUN)[1]) sha256 = util.deser_uint256(s) level = ord(s.read(1)) index = util.deser_varint(s) generations = ord(s.read(1)) length = util.deser_varint(s) flags = util.deser_varint(s) if flags: raise NotImplementedError run = [s.read(32) for i in range(length)] result = self.merkles[sha256].checkaddrun(level, index, generations, length, run) if not result: print "Failed to add from peer=%s: l=%i i=%i g=%i h=%s" % (str(peer), level, index, generations, util.ser_uint256(sha256)[::-1].encode('hex')) debuglog('btnet', "Failed to add from peer=%s: l=%i i=%i g=%i h=%s" % (str(peer), level, index, generations, util.ser_uint256(sha256)[::-1].encode('hex'))) else: self.maybe_update_peers(sha256)
def connect_finish(self, t, addr, magic): '''This is called whenever the other side has accepted our connection request. Step 2 of three-way handshake. Equivalent to TCP syn-sent -> established transition. ''' # addr should be (IP address, port) since it is from recvfrom() if addr in self.syn_sent: payload = t.split(BTMessage.MSG_CONNECT_ACK, 1)[1] key = payload[HASH_SIZE:] peer = self.syn_sent[addr] if key in peer.unacknowledged: del peer.unacknowledged[key] del self.syn_sent[addr] if (addr, magic) in self.syn_received: del self.syn_received[(addr, magic)] self.peer_adder.addnode(peer, magic) # TODO: do we need sequence here? peer.send_message(BTMessage.MSG_ACK + calculate_hash(payload[0:HASH_SIZE])) else: debuglog('btnet', "Got malformed CONNECT-ACK from %s:%i" % addr) else: debuglog('btnet', "Got unexpected CONNECT-ACK from %s:%i" % addr)
def recv_nodes(self, data, peer): s = StringIO.StringIO(data.split(BTMessage.MSG_RUN)[1]) sha256 = util.deser_uint256(s) level = ord(s.read(1)) index = util.deser_varint(s) generations = ord(s.read(1)) length = util.deser_varint(s) flags = util.deser_varint(s) if flags: raise NotImplementedError run = [s.read(32) for i in range(length)] result = self.merkles[sha256].checkaddrun(level, index, generations, length, run) if not result: print "Failed to add from peer=%s: l=%i i=%i g=%i h=%s" % ( str(peer), level, index, generations, util.ser_uint256(sha256)[::-1].encode('hex')) debuglog( 'btnet', "Failed to add from peer=%s: l=%i i=%i g=%i h=%s" % (str(peer), level, index, generations, util.ser_uint256(sha256)[::-1].encode('hex'))) else: self.maybe_update_peers(sha256)
def send_nodes(self, peer, sha256, level, index, generations, flags): if not sha256 in self.merkles: debuglog('btnet', 'peer %s wants a block that we don\'t know about: %s' % (str(peer), util.ser_uint256(sha256)[::-1].encode('hex'))) peer.send_message(BTMessage.MSG_MISSING_BLOCK + util.ser_uint256(sha256) + chr(level) + util.ser_varint(index) + chr(generations)) return if not self.merkles[sha256].state.hasdescendants(level, index, generations): debuglog('btnet', 'peer %s wants nodes that we don\'t know about: l=%i i=%i g=%i h=%s' % (str(peer), leve, index, generations, util.ser_uint256(sha256)[::-1].encode('hex'))) peer.send_message(BTMessage.MSG_MISSING_NODES + util.ser_uint256(sha256) + chr(level) + util.ser_varint(index) + chr(generations)) return run = self.merkles[sha256].getrun(level, index, generations) assert type(run[0]) == str and len(run[0]) == 32 # Just checking to make sure that merkles stores the serialized str version of the hash, since I forgot flags = 0 if flags: raise NotImplementedError msg = BTMessage.MSG_RUN + util.ser_uint256(sha256) + chr(level) + util.ser_varint(index) + chr(generations) + util.ser_varint(len(run)) + util.ser_varint(flags) + ''.join(run) if len(msg) > peer.MTU: debuglog('btnet', 'MSG_RUN has length %i which exceeds peer %s\'s max MTU of %i' % (len(msg), str(peer), peer.MTU)) peer.send_message(msg)
def process_message(self, packet, addr): m = btnet.BTMessage.deserialize(packet) peer = None if (m.magic, addr) in self.magic_map: peer = self.magic_map[(m.magic, addr)] debuglog('btnet', "Received from %s: %s" % (':'.join(map(str, addr)), str(m))) try: if not peer: # Not connected yet self.peer_manager.process_message(m, addr) else: # connected if m.payload.startswith(BTMessage.MSG_DISCONNECT): self.remnode(peer) if m.payload.startswith(BTMessage.MSG_HEADER): self.recv_header(m.payload, peer) if m.payload.startswith(BTMessage.MSG_MULTIPLE): self.recv_multiple(m.payload, addr) if m.payload.startswith(BTMessage.MSG_BLOCKSTATE): self.recv_blockstate(m.payload, peer) if m.payload.startswith(BTMessage.MSG_ACK): self.recv_ack(m, peer) if m.sequence: if (m.magic, addr) in self.magic_map: peer = self.magic_map[(m.magic, addr)] self.peer_manager.send_ack(m, peer.low_level_peer) except: debuglog('btnet', 'Malformed UDP message or parsing error') debuglog('btnet', traceback.format_exc()) traceback.print_exc()
def send_nodes(self, peer, sha256, level, index, generations, flags): if not sha256 in self.merkles: debuglog( 'btnet', 'peer %s wants a block that we don\'t know about: %s' % (str(peer), util.ser_uint256(sha256)[::-1].encode('hex'))) peer.send_message(BTMessage.MSG_MISSING_BLOCK + util.ser_uint256(sha256) + chr(level) + util.ser_varint(index) + chr(generations)) return if not self.merkles[sha256].state.hasdescendants( level, index, generations): debuglog( 'btnet', 'peer %s wants nodes that we don\'t know about: l=%i i=%i g=%i h=%s' % (str(peer), leve, index, generations, util.ser_uint256(sha256)[::-1].encode('hex'))) peer.send_message(BTMessage.MSG_MISSING_NODES + util.ser_uint256(sha256) + chr(level) + util.ser_varint(index) + chr(generations)) return run = self.merkles[sha256].getrun(level, index, generations) assert type(run[0]) == str and len( run[0] ) == 32 # Just checking to make sure that merkles stores the serialized str version of the hash, since I forgot flags = 0 if flags: raise NotImplementedError msg = BTMessage.MSG_RUN + util.ser_uint256(sha256) + chr( level) + util.ser_varint(index) + chr( generations) + util.ser_varint( len(run)) + util.ser_varint(flags) + ''.join(run) if len(msg) > peer.MTU: debuglog( 'btnet', 'MSG_RUN has length %i which exceeds peer %s\'s max MTU of %i' % (len(msg), str(peer), peer.MTU)) peer.send_message(msg)
def process_message(self, packet, addr): m = btnet.BTMessage.deserialize(packet) peer = None if (m.magic, addr) in self.magic_map: peer = self.magic_map[(m.magic, addr)] debuglog('btnet', "Received from %s: %s" % (':'.join(map(str, addr)), str(m))) try: if not peer: # Not connected yet self.peer_manager.process_message(m, addr) else: # connected if m.payload.startswith(BTMessage.MSG_DISCONNECT): self.remnode(peer) if m.payload.startswith(BTMessage.MSG_HEADER): self.recv_header(m.payload, peer) if m.payload.startswith(BTMessage.MSG_MULTIPLE): self.recv_multiple(m.payload, addr) if m.payload.startswith(BTMessage.MSG_BLOCKSTATE): self.recv_blockstate(m.payload, peer) if m.payload.startswith(BTMessage.MSG_ACK): self.recv_ack(m, peer) if m.payload.startswith(BTMessage.MSG_TX): self.recv_tx(m.payload, peer) # args? if m.payload.startswith(BTMessage.MSG_REQUEST_TX): self.send_tx(m.payload, peer) # args? # need request_tx func. receive tx req and tx msg # asking for specific tx? by txhash, or by blockhash and tx index, # or multiple tx by list of tx indices (offsets). 3 tx in row, 3tx [0,0,0]. would be bandwidth efficient # need mempool for txs. dict of tx hashes to tx obj? tx obj from mininode, or other class we write on top # would want to add salted short hashes -- eventually # receive req: check mempool. # how would you check with req by offset or index? # go into your block db, find that block, find hash that goes at that index, use that to get tx out of mempool # Test: fill mempool with data from getblocktemplate, other nodes can req tx from it, they can fill their mempools, get complete blocks # although don't have logic for which parts of merkle tree to req.... # write hardcoded thing that sends tx from one to another, check if its received at 2nd peer if m.payload.startswith(BTMessage.MSG_REQUEST_NODES): self.recv_node_request(m.payload, peer) if m.payload.startswith(BTMessage.MSG_RUN): self.recv_nodes(m.payload, peer) if m.payload.startswith(BTMessage.MSG_REQ_TXCOUNT): self.handle_txcount_req(m.payload, peer) if m.payload.startswith(BTMessage.MSG_TXCOUNT_PROOF): self.recv_txcount_proof(m.payload, peer) if m.payload.startswith(BTMessage.MSG_MISSING_BLOCK): debuglog( 'btnet', "MSG_MISSING_BLOCK received, but we can't parse it yet. Payload: %s" % m.payload) if m.payload.startswith(BTMessage.MSG_MISSING_NODES): debuglog( 'btnet', "MSG_MISSING_NODES received, but we can't parse it yet. Payload: %s" % m.payload) if m.sequence: if (m.magic, addr) in self.magic_map: peer = self.magic_map[(m.magic, addr)] self.peer_manager.send_ack(m, peer.low_level_peer) except: debuglog('btnet', 'Malformed UDP message or parsing error') debuglog('btnet', traceback.format_exc()) traceback.print_exc()
def process_message(self, packet, addr): m = btnet.BTMessage.deserialize(packet) peer = None if (m.magic, addr) in self.magic_map: peer = self.magic_map[(m.magic, addr)] debuglog('btnet', "Received from %s: %s" % (':'.join(map(str, addr)), str(m))) try: if not peer: # Not connected yet self.peer_manager.process_message(m, addr) else: # connected if m.payload.startswith(BTMessage.MSG_DISCONNECT): self.remnode(peer) if m.payload.startswith(BTMessage.MSG_HEADER): self.recv_header(m.payload, peer) if m.payload.startswith(BTMessage.MSG_MULTIPLE): self.recv_multiple(m.payload, addr) if m.payload.startswith(BTMessage.MSG_BLOCKSTATE): self.recv_blockstate(m.payload, peer) if m.payload.startswith(BTMessage.MSG_ACK): self.recv_ack(m, peer) if m.payload.startswith(BTMessage.MSG_TX): self.recv_tx(m.payload, peer) # args? if m.payload.startswith(BTMessage.MSG_REQUEST_TX): self.send_tx(m.payload, peer) # args? # need request_tx func. receive tx req and tx msg # asking for specific tx? by txhash, or by blockhash and tx index, # or multiple tx by list of tx indices (offsets). 3 tx in row, 3tx [0,0,0]. would be bandwidth efficient # need mempool for txs. dict of tx hashes to tx obj? tx obj from mininode, or other class we write on top # would want to add salted short hashes -- eventually # receive req: check mempool. # how would you check with req by offset or index? # go into your block db, find that block, find hash that goes at that index, use that to get tx out of mempool # Test: fill mempool with data from getblocktemplate, other nodes can req tx from it, they can fill their mempools, get complete blocks # although don't have logic for which parts of merkle tree to req.... # write hardcoded thing that sends tx from one to another, check if its received at 2nd peer if m.payload.startswith(BTMessage.MSG_REQUEST_NODES): self.recv_node_request(m.payload, peer) if m.payload.startswith(BTMessage.MSG_RUN): self.recv_nodes(m.payload, peer) if m.payload.startswith(BTMessage.MSG_REQ_TXCOUNT): self.handle_txcount_req(m.payload, peer) if m.payload.startswith(BTMessage.MSG_TXCOUNT_PROOF): self.recv_txcount_proof(m.payload, peer) if m.payload.startswith(BTMessage.MSG_MISSING_BLOCK): debuglog('btnet', "MSG_MISSING_BLOCK received, but we can't parse it yet. Payload: %s" % m.payload) if m.payload.startswith(BTMessage.MSG_MISSING_NODES): debuglog('btnet', "MSG_MISSING_NODES received, but we can't parse it yet. Payload: %s" % m.payload) if m.sequence: if (m.magic, addr) in self.magic_map: peer = self.magic_map[(m.magic, addr)] self.peer_manager.send_ack(m, peer.low_level_peer) except: debuglog('btnet', 'Malformed UDP message or parsing error') debuglog('btnet', traceback.format_exc()) traceback.print_exc()