def test_handle_chain_state_works(self):
"""
When a peer reports its chain state, we update the timestamp to reflect the time we received the message.
Then we update our _peer_node_status.
"""
chain_state_data = make_node_chain_state()
chain_state_message = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.CHAINSTATE,
chainStateData=chain_state_data)
channel = make_channel()
# The P2PPeerManager should update its hash table with the info from chain_state_data.
# Unfortunately it also updates the timestamp so we cannot simply compare the objects,
# we have to compare the fields in the object excluding the timestamp.
self.peer_manager.handle_chain_state(channel, chain_state_message)
self.assertEqual(
self.peer_manager._peer_node_status[channel].block_number,
chain_state_data.block_number)
self.assertEqual(
self.peer_manager._peer_node_status[channel].header_hash,
chain_state_data.header_hash)
self.assertEqual(
self.peer_manager._peer_node_status[channel].cumulative_difficulty,
chain_state_data.cumulative_difficulty)
def handle_version(self, source, message: qrllegacy_pb2.LegacyMessage):
"""
Version
If version is empty, it sends the version & genesis_prev_headerhash.
Otherwise, processes the content of data.
In case of mismatches, it disconnects from the peer
"""
self._validate_message(message, qrllegacy_pb2.LegacyMessage.VE)
if not message.veData.version:
msg = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.VE,
veData=qrllegacy_pb2.VEData(version=config.dev.version,
genesis_prev_hash=config.dev.genesis_prev_headerhash,
rate_limit=config.user.peer_rate_limit))
source.send(msg)
return
logger.info('%s version: %s | genesis prev_headerhash %s',
source.peer_ip,
message.veData.version,
message.veData.genesis_prev_hash)
if source.last_rate_limit_update > 0:
source.factory.ban_peer(source) # Peer Breaking protocol, disconnect & Ban
source.rate_limit = min(config.user.peer_rate_limit, message.veData.rate_limit)
if message.veData.genesis_prev_hash != config.dev.genesis_prev_headerhash:
logger.warning('%s genesis_prev_headerhash mismatch', source.connection_id)
logger.warning('Expected: %s', config.dev.genesis_prev_headerhash)
logger.warning('Found: %s', message.veData.genesis_prev_hash)
source.loseConnection()
def _parse_buffer(self, total_read):
# FIXME: This parsing/wire protocol needs to be replaced
"""
>>> from pyqrllib.pyqrllib import hstr2bin
>>> p=P2PProtocol()
>>> p._buffer = bytes(hstr2bin('000000191a170a0776657273696f6e120c67656e657369735f68617368'+ \
'000000191a170a0776657273696f6e120c67656e657369735f68617368'))
>>> messages = p._parse_buffer([0])
>>> len(list(messages))
2
"""
while self._buffer:
# FIXME: This is not the final implementation, it is just a minimal implementation for refactoring
if len(self._buffer) < 4:
# Buffer is still incomplete as it doesn't have message size
return
chunk_size_raw = self._buffer[:4]
chunk_size = struct.unpack('>L', chunk_size_raw)[0] # is m length encoded correctly?
# FIXME: There is no limitation on the buffer size or timeout
if len(self._buffer) < chunk_size:
# Buffer is still incomplete as it doesn't have message
return
try:
message_raw = self._buffer[4:4 + chunk_size]
message = qrllegacy_pb2.LegacyMessage()
message.ParseFromString(message_raw)
yield message
except Exception as e:
logger.warning("Problem parsing message. Skipping")
finally:
self._buffer = self._buffer[4 + chunk_size:]
total_read[0] += 4 + chunk_size
def dataReceived(self, data: bytes) -> None:
self._buffer += data
total_read = len(self._buffer)
if total_read > config.dev.max_bytes_out:
logger.warning('Disconnecting peer %s', self.peer)
logger.warning('Buffer Size %s', len(self._buffer))
self.loseConnection()
return
read_bytes = [0]
msg = None
for msg in self._parse_buffer(read_bytes):
self.update_counters()
self.in_counter += 1
if self.in_counter > self.rate_limit * IN_FACTOR:
logger.warning("Rate Limit hit by %s %s", self.peer.ip, self.peer.port)
self.peer_manager.ban_channel(self)
return
if self._valid_message_count < config.dev.trust_min_msgcount * 2:
# Avoid overflows
self._valid_message_count += 1
self._observable.notify(msg)
if msg is not None and read_bytes[0] and msg.func_name != qrllegacy_pb2.LegacyMessage.P2P_ACK:
p2p_ack = qrl_pb2.P2PAcknowledgement(bytes_processed=read_bytes[0])
msg = qrllegacy_pb2.LegacyMessage(func_name=qrllegacy_pb2.LegacyMessage.P2P_ACK,
p2pAckData=p2p_ack)
self.send(msg)
def dataReceived(self, data: bytes) -> None:
self._buffer += data
total_read = len(self._buffer)
if total_read > config.dev.max_bytes_out:
logger.warning('Disconnecting peer %s', self.addr_remote)
logger.warning('Buffer Size %s', len(self._buffer))
self.loseConnection()
read_bytes = [0]
for msg in self._parse_buffer(read_bytes):
self.update_counters()
self.in_counter += 1
if self.in_counter > self.rate_limit:
self.factory.ban_peer(self)
self._observable.notify(msg)
if read_bytes[
0] and msg.func_name != qrllegacy_pb2.LegacyMessage.P2P_ACK:
p2p_ack = qrl_pb2.P2PAcknowledgement(bytes_processed=read_bytes[0])
msg = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.P2P_ACK,
p2pAckData=p2p_ack)
self.send(msg)
def send_get_headerhash_list(self, current_block_height):
start_blocknumber = max(0, current_block_height - config.dev.reorg_limit)
node_header_hash = qrl_pb2.NodeHeaderHash(block_number=start_blocknumber,
headerhashes=[])
msg = qrllegacy_pb2.LegacyMessage(func_name=qrllegacy_pb2.LegacyMessage.HEADERHASHES,
nodeHeaderHash=node_header_hash)
self.send(msg)
def send_node_chain_state(dest_channel,
node_chain_state: qrl_pb2.NodeChainState):
# FIXME: Not sure this belongs to peer management
msg = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.CHAINSTATE,
chainStateData=node_chain_state)
dest_channel.send(msg)
def handle_block_height(self, source,
message: qrllegacy_pb2.LegacyMessage):
"""
Sends / Receives Blockheight
:param source:
:param message:
:return:
"""
if message.bhData.block_number == 0:
block = source.factory.get_last_block()
cumulative_difficulty = source.factory.get_cumulative_difficulty()
if block.block_number == 0:
return
bhdata = qrl_pb2.BlockHeightData(
block_number=block.block_number,
block_headerhash=block.headerhash,
cumulative_difficulty=bytes(cumulative_difficulty))
msg = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.BH, bhData=bhdata)
source.send(msg)
return
try:
UInt256ToString(message.chainStateData.cumulative_difficulty)
except ValueError:
logger.warning('Invalid Block Height Data')
source.loseConnection()
return
source.factory.update_peer_blockheight(
source.addr_remote, message.bhData.block_number,
message.bhData.block_headerhash,
message.bhData.cumulative_difficulty)
def send_sync(self, synced=False):
state_str = ''
if synced:
state_str = 'Synced'
msg = qrllegacy_pb2.LegacyMessage(func_name=qrllegacy_pb2.LegacyMessage.SYNC,
syncData=qrllegacy_pb2.SYNCData(state=state_str))
self.send(msg)
def test_wrap_message_works(self):
veData = qrllegacy_pb2.VEData(version="version",
genesis_prev_hash=b'genesis_hash')
msg = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.VE, veData=veData)
self.assertEqual(
'000000191a170a0776657273696f6e120c67656e657369735f68617368',
bin2hstr(P2PProtocol._wrap_message(msg)))
def test_handle_sync_unsynced(self):
"""
If the message says anything else: the peer is unsynced.
"""
sync_message = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.SYNC,
syncData=qrllegacy_pb2.SYNCData(state='Unsynced'))
channel = make_channel()
self.peer_manager.handle_sync(channel, sync_message)
def test_handle_sync(self):
"""
If a message comes in saying 'Synced': the peer is synced.
"""
sync_message = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.SYNC,
syncData=qrllegacy_pb2.SYNCData(state='Synced'))
channel = make_channel()
self.peer_manager.handle_sync(channel, sync_message)
def send_fetch_block(self, block_idx):
"""
Fetch Block n
Sends request for the block number n.
:return:
"""
logger.info('<<<Fetching block: %s from %s', block_idx, self.peer)
msg = qrllegacy_pb2.LegacyMessage(func_name=qrllegacy_pb2.LegacyMessage.FB,
fbData=qrllegacy_pb2.FBData(index=block_idx))
self.send(msg)
def _parse_buffer(self, total_read):
# FIXME: This parsing/wire protocol needs to be replaced
"""
>>> from pyqrllib.pyqrllib import hstr2bin
>>> p=P2PProtocol()
>>> p._buffer = bytes(hstr2bin('000000191a170a0776657273696f6e120c67656e657369735f68617368'+ \
'000000191a170a0776657273696f6e120c67656e657369735f68617368'))
>>> messages = p._parse_buffer([0])
>>> len(list(messages))
2
"""
chunk_size = 0
while self._buffer:
if len(self._buffer) < 5:
# Buffer is still incomplete as it doesn't have message size
return
ignore_skip = False
try:
chunk_size_raw = self._buffer[:4]
chunk_size = struct.unpack(
'>L', chunk_size_raw)[0] # is m length encoded correctly?
if chunk_size <= 0:
logger.debug("<X< %s", bin2hstr(self._buffer))
raise Exception("Invalid chunk size <= 0")
if chunk_size > config.dev.message_buffer_size:
raise Exception("Invalid chunk size > message_buffer_size")
if len(
self._buffer
) - 4 < chunk_size: # As 4 bytes includes chunk_size_raw
ignore_skip = True # Buffer is still incomplete as it doesn't have message so skip moving buffer
return
message_raw = self._buffer[4:4 + chunk_size]
message = qrllegacy_pb2.LegacyMessage()
message.ParseFromString(message_raw)
yield message
except Exception as e: # no qa
logger.warning(
"Problem parsing message. Banning+Dropping connection")
logger.exception(e)
self.peer_manager.ban_channel(self)
finally:
if not ignore_skip:
skip = 4 + chunk_size
self._buffer = self._buffer[skip:]
total_read[0] += skip
def get(self, msg_type,
msg_hash: bytes) -> Optional[qrllegacy_pb2.LegacyMessage]:
if not self.contains(msg_hash, msg_type):
return None
msg = self._hash_msg[msg_hash].msg
data = qrllegacy_pb2.LegacyMessage(**{
'func_name': msg_type,
self.services_arg[msg_type]: msg
})
return data
def test_handle_sync(self):
"""
If a message comes in saying 'Synced': the peer is synced.
"""
sync_message = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.SYNC,
syncData=qrllegacy_pb2.SYNCData(state='Synced'))
channel = make_channel()
self.peer_manager.handle_sync(channel, sync_message)
channel.factory.set_peer_synced.assert_called_once_with(channel, True)
def test_handle_p2p_acknowledgement_negative_bytes_processed(self):
"""
If we sent 10 bytes to the peer, but the peer says it processed 20 bytes, disconnect the peer.
"""
ack = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.P2P_ACK,
p2pAckData=qrl_pb2.P2PAcknowledgement(bytes_processed=20))
channel = make_channel()
channel.bytes_sent = 10
self.peer_manager.handle_p2p_acknowledgement(channel, ack)
channel.loseConnection.assert_called_once_with()
def test_handle_p2p_acknowledgement(self):
"""
Once an acknowledgement is received from a peer, we can update rate counters and send the next message.
"""
ack = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.P2P_ACK,
p2pAckData=qrl_pb2.P2PAcknowledgement(bytes_processed=15))
channel = make_channel()
channel.bytes_sent = 20
self.peer_manager.handle_p2p_acknowledgement(channel, ack)
channel.send_next.assert_called_once_with()
def send_peer_list(self):
"""
Get Peers
Sends the peers list.
:return:
"""
remote_peers = self.factory.get_connected_peer_addrs()
logger.debug('<<< Sending connected peers to %s [%s]', self.addr_remote, remote_peers)
msg = qrllegacy_pb2.LegacyMessage(func_name=qrllegacy_pb2.LegacyMessage.PL,
plData=qrllegacy_pb2.PLData(peer_ips=remote_peers))
self.send(msg)
def test_handle_version_wrong_genesis_prev_headerhash(self):
"""
If the genesis_prev_headerhash is different, the nodes should disconnect from each other.
"""
channel = make_channel()
message = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.VE,
veData=qrllegacy_pb2.VEData(
version=config.dev.version,
genesis_prev_hash=b'TEST123',
rate_limit=config.user.peer_rate_limit))
self.peer_manager.handle_version(channel, message)
channel.loseConnection.assert_any_call()
def send_peer_list(self):
"""
Get Peers
Sends the peers list.
:return:
"""
trusted_peers = self.peer_manager.trusted_addresses
logger.debug('<<< Sending connected peers to %s [%s]', self.peer, trusted_peers)
msg = qrllegacy_pb2.LegacyMessage(func_name=qrllegacy_pb2.LegacyMessage.PL,
plData=qrllegacy_pb2.PLData(peer_ips=trusted_peers,
public_port=config.user.p2p_public_port))
self.send(msg)
def handle_node_headerhash(self, source, message: qrllegacy_pb2.LegacyMessage):
"""
Sends/Receives NodeHeaderHashes
:param source:
:param message:
:return:
"""
if len(message.nodeHeaderHash.headerhashes) == 0:
node_headerhash = source.factory.get_headerhashes(message.nodeHeaderHash.block_number)
msg = qrllegacy_pb2.LegacyMessage(func_name=qrllegacy_pb2.LegacyMessage.HEADERHASHES,
nodeHeaderHash=node_headerhash)
source.send(msg)
else:
source.factory.compare_and_sync(source, message.nodeHeaderHash)
def handle_fetch_block(self, source, message: qrllegacy_pb2.LegacyMessage): # Fetch Request for block
"""
Fetch Block
Sends the request for the block.
:return:
"""
P2PBaseObserver._validate_message(message, qrllegacy_pb2.LegacyMessage.FB)
block_number = message.fbData.index
logger.info(' Request for %s by %s', block_number, source.connection_id)
if 0 < block_number <= source.factory.chain_height:
block = source.factory.get_block(block_number)
msg = qrllegacy_pb2.LegacyMessage(func_name=qrllegacy_pb2.LegacyMessage.PB,
pbData=qrllegacy_pb2.PBData(block=block.pbdata))
source.send(msg)
def test_handle_version_empty_version_message(self):
"""
If the incoming version message has an empty version field, then send another version request.
That message should have the node's version in it.
"""
channel = make_channel()
message = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.VE,
veData=qrllegacy_pb2.VEData(
version='',
genesis_prev_hash=config.user.genesis_prev_headerhash,
rate_limit=config.user.peer_rate_limit))
self.peer_manager.handle_version(channel, message)
self.assertEqual(channel.send.call_args[0][0].veData.version,
config.dev.version)
def test_handle_peer_list_works(self, logger):
"""
Heavy error testing should be done in combine_peer_lists() and extend_known_peers(), which this fx uses.
"""
peer_list_message = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.PL,
plData=qrllegacy_pb2.PLData(
peer_ips={'127.0.0.3:5000', '127.0.0.4:5001'},
public_port=9000))
channel = make_channel()
channel.host = IPMetadata('187.0.0.1', 9000)
channel.peer = IPMetadata('187.0.0.2', 9000)
channel.ip_public_port = '187.0.0.1:9000'
# handle_peer_list() will call extend_known_peers(), so we gotta mock it out. It's tested elsewhere anyway.
self.peer_manager.handle_peer_list(channel, peer_list_message)
def request_full_message(self, mr_data: qrllegacy_pb2.MRData):
"""
Request Full Message
This function request for the full message against,
the Message Receipt received.
:return:
"""
# FIXME: Again, breaking encasulation
# FIXME: Huge amount of lookups in dictionaries
msg_hash = mr_data.hash
if msg_hash in self.master_mr._hash_msg:
if msg_hash in self.master_mr.requested_hash:
del self.master_mr.requested_hash[msg_hash]
return
if msg_hash not in self.master_mr.requested_hash:
return
peers_list = self.master_mr.requested_hash[
msg_hash].peers_connection_list
message_request = self.master_mr.requested_hash[msg_hash]
for peer in peers_list:
if peer in message_request.already_requested_peers:
continue
message_request.already_requested_peers.append(peer)
msg = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.SFM,
mrData=qrllegacy_pb2.MRData(hash=mr_data.hash,
type=mr_data.type))
peer.send(msg)
call_later_obj = reactor.callLater(
config.dev.message_receipt_timeout, self.request_full_message,
mr_data)
message_request.callLater = call_later_obj
return
# If execution reach to this line, then it means no peer was able to provide
# Full message for this hash thus the hash has to be deleted.
# Moreover, negative points could be added to the peers, for this behavior
if msg_hash in self.master_mr.requested_hash:
del self.master_mr.requested_hash[msg_hash]
def test_handle_version(self):
"""
When a version message arrives from a peer, and all else is normal:
A version request message was not sent to the peer.
The peer is not banned.
P2PProtocol.loseConnection() is not called.
"""
channel = make_channel()
message = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.VE,
veData=qrllegacy_pb2.VEData(
version=config.dev.version,
genesis_prev_hash=config.user.genesis_prev_headerhash,
rate_limit=config.user.peer_rate_limit))
self.peer_manager.handle_version(channel, message)
channel.peer_manager.ban_channel.assert_not_called()
channel.loseConnection.assert_not_called()
def test_handle_sync_blank(self):
"""
If the message says '': the peer doesn't know, and isn't synced. We tell it that we are synced.
"""
sync_message = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.SYNC,
syncData=qrllegacy_pb2.SYNCData(state=''))
channel = make_channel()
# But if we ourselves aren't synced, then we cannot say if anybody else is synced.
channel.factory.synced = False
self.peer_manager.handle_sync(channel, sync_message)
channel.send_sync.assert_not_called()
# If we are synced, then we can tell other nodes we are synced.
channel.factory.synced = True
self.peer_manager.handle_sync(channel, sync_message)
channel.send_sync.assert_called_once_with(synced=True)
def test_handle_peer_list_works(self, logger):
"""
Heavy error testing should be done in get_valid_peers() and update_peer_addresses(), which this fx uses.
"""
peer_list_message = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.PL,
plData=qrllegacy_pb2.PLData(
peer_ips={'127.0.0.3:5000', '127.0.0.4:5001'},
public_port=9000))
channel = make_channel()
channel.host_ip = '187.0.0.1'
channel.peer_ip = '187.0.0.2'
# handle_peer_list() will call update_peer_addresses(), so we gotta mock it out. It's tested elsewhere anyway.
self.peer_manager.update_peer_addresses = Mock(
autospec=P2PPeerManager.update_peer_addresses)
self.peer_manager.handle_peer_list(channel, peer_list_message)
self.peer_manager.update_peer_addresses.assert_called_once_with(
{"{0}:{1}".format(channel.peer_ip, 9000)})
def test_bad_tx(self):
source = Mock()
source.factory = Mock()
source.factory.master_mr = Mock()
source.factory.master_mr.isRequested = Mock()
source.factory.add_unprocessed_txn = Mock()
channel = Observable(source)
self.tx_manager = P2PTxManagement()
self.tx_manager.new_channel(channel)
tx = SlaveTransaction.create([], [], 1, bytes(100))
event = qrllegacy_pb2.LegacyMessage(func_name=qrllegacy_pb2.LegacyMessage.TX,
txData=tx.pbdata)
channel.notify(event, force_delivery=True)
source.factory.master_mr.isRequested.assert_not_called()
source.factory.add_unprocessed_txn.assert_not_called()