def HandleInvMessage(self, payload):
"""
Process a block header inventory payload.
Args:
inventory (neo.Network.Payloads.InvPayload):
"""
if self.sync_mode != MODE_MAINTAIN:
return
inventory = IOHelper.AsSerializableWithType(
payload, 'neo.Network.Payloads.InvPayload.InvPayload')
if inventory.Type == InventoryType.BlockInt:
ok_hashes = []
for hash in inventory.Hashes:
hash = hash.encode('utf-8')
if hash not in self.myblockrequests and hash not in BC.Default(
).BlockRequests:
ok_hashes.append(hash)
BC.Default().BlockRequests.add(hash)
self.myblockrequests.add(hash)
if len(ok_hashes):
# logger.info("OK HASHES, get data %s " % ok_hashes)
message = Message("getdata",
InvPayload(InventoryType.Block, ok_hashes))
self.SendSerializedMessage(message)
elif inventory.Type == InventoryType.TXInt:
pass
elif inventory.Type == InventoryType.ConsensusInt:
pass
def HandleGetBlocksMessageReceived(self, payload):
"""
Process a GetBlocksPayload payload.
Args:
payload (neo.Network.Payloads.GetBlocksPayload):
"""
if not self.leader.ServiceEnabled:
return
inventory = IOHelper.AsSerializableWithType(
payload, 'neo.Network.Payloads.GetBlocksPayload.GetBlocksPayload')
if not BC.Default().GetHeader(inventory.HashStart):
self.Log("Hash %s not found %s " % inventory.HashStart)
return
hashes = []
hcount = 0
hash = inventory.HashStart
while hash != inventory.HashStop and hcount < 500:
hash = BC.Default().GetNextBlockHash(hash)
if hash is None:
break
hashes.append(hash)
hcount += 1
if hcount > 0:
self.Log("sending inv hashes! %s " % hashes)
self.SendSerializedMessage(
Message('inv',
InvPayload(type=InventoryType.Block, hashes=hashes)))
def DoAskForMoreBlocks(self):
hashes = []
hashstart = BC.Default().Height + 1
current_header_height = BC.Default().HeaderHeight + 1
do_go_ahead = False
if BC.Default().BlockSearchTries > 100 and len(BC.Default().BlockRequests) > 0:
do_go_ahead = True
first = None
while hashstart < current_header_height and len(hashes) < self.leader.BREQPART:
hash = BC.Default().GetHeaderHash(hashstart)
if not do_go_ahead:
if hash is not None and hash not in BC.Default().BlockRequests \
and hash not in self.myblockrequests:
if not first:
first = hashstart
BC.Default().BlockRequests.add(hash)
self.myblockrequests.add(hash)
hashes.append(hash)
else:
if not first:
first = hashstart
BC.Default().BlockRequests.add(hash)
self.myblockrequests.add(hash)
hashes.append(hash)
hashstart += 1
self.Log("asked for more blocks ... %s thru %s (%s blocks) stale count %s BCRLen: %s " % (first, hashstart, len(hashes), BC.Default().BlockSearchTries, len(BC.Default().BlockRequests)))
if len(hashes) > 0:
message = Message("getdata", InvPayload(InventoryType.Block, hashes))
reactor.callInThread(self.SendSerializedMessage, message)
def SendVersion(self):
"""Send our client version."""
m = Message(
"version",
VersionPayload(settings.NODE_PORT, self.remote_nodeid,
settings.VERSION_NAME))
self.SendSerializedMessage(m)
def Relay(self, inventory):
inventory = InvPayload(type=inventory.InventoryType, hashes=[inventory.Hash.ToBytes()])
m = Message("inv", inventory)
self.SendSerializedMessage(m)
return True
def HandleGetBlocksMessageReceived(self, payload):
"""
Process a GetBlocksPayload payload.
Args:
payload (neo.Network.Payloads.GetBlocksPayload):
"""
if not self.leader.ServiceEnabled:
return
inventory = IOHelper.AsSerializableWithType(
payload, 'neo.Network.Payloads.GetBlocksPayload.GetBlocksPayload')
if not inventory:
return
blockchain = BC.Default()
hash = inventory.HashStart[0]
if not blockchain.GetHeader(hash):
return
hashes = []
hcount = 0
while hash != inventory.HashStop and hcount < 500:
hash = blockchain.GetNextBlockHash(hash)
if hash is None:
break
hashes.append(hash)
hcount += 1
if hcount > 0:
self.SendSerializedMessage(
Message('inv',
InvPayload(type=InventoryType.Block, hashes=hashes)))
def DoAskForMoreBlocks(self):
hashes = []
hashstart = BC.Default().Height + 1
if BC.Default().BlockSearchTries > 400 and len(BC.Default().BlockRequests) > 0:
self.leader.ResetBlockRequestsAndCache()
first=None
while hashstart < BC.Default().HeaderHeight and len(hashes) < self.leader.BREQPART:
hash = BC.Default().GetHeaderHash(hashstart)
if hash is not None and not hash in BC.Default().BlockRequests and not hash in self.myblockrequests:
if not first:
first = hashstart
BC.Default().BlockRequests.add(hash)
self.myblockrequests.add(hash)
hashes.append(hash)
hashstart += 1
self.Log("asked for more blocks ... %s thru %s stale count %s " % (first,hashstart, BC.Default().BlockSearchTries))
if len(hashes) > 0:
message = Message("getdata", InvPayload(InventoryType.Block, hashes))
self.SendSerializedMessage(message)
else:
reactor.callLater(5, self.AskForMoreBlocks)
def HandleGetDataMessageReceived(self, payload):
"""
Process a InvPayload payload.
Args:
payload (neo.Network.Inventory):
"""
inventory = IOHelper.AsSerializableWithType(payload, 'neo.Network.Payloads.InvPayload.InvPayload')
for hash in inventory.Hashes:
hash = hash.encode('utf-8')
item = None
# try to get the inventory to send from relay cache
if hash in self.leader.RelayCache.keys():
item = self.leader.RelayCache[hash]
if item:
if inventory.Type == int.from_bytes(InventoryType.TX, 'little'):
message = Message(command='tx', payload=item, print_payload=True)
self.SendSerializedMessage(message)
elif inventory.Type == int.from_bytes(InventoryType.Block, 'little'):
logger.info("handle block!")
elif inventory.Type == int.from_bytes(InventoryType.Consensus, 'little'):
logger.info("handle consensus")
def HandleGetHeadersMessageReceived(self, payload):
if not self.leader.ServiceEnabled:
return
inventory = IOHelper.AsSerializableWithType(payload, 'neo.Network.Payloads.GetBlocksPayload.GetBlocksPayload')
if not inventory:
return
blockchain = BC.Default()
hash = inventory.HashStart[0]
if hash is None or hash == inventory.HashStop:
self.Log("getheaders: Hash {} not found or hashstop reached".format(inventory.HashStart))
return
headers = []
header_count = 0
while hash != inventory.HashStop and header_count < 2000:
hash = blockchain.GetNextBlockHash(hash)
if not hash:
break
headers.append(blockchain.GetHeader(hash))
header_count += 1
if header_count > 0:
self.SendSerializedMessage(Message('headers', HeadersPayload(headers=headers)))
def HandleGetDataMessageReceived(self, payload):
"""
Process a InvPayload payload.
Args:
payload (neo.Network.Inventory):
"""
inventory = IOHelper.AsSerializableWithType(
payload, 'neo.Network.Payloads.InvPayload.InvPayload')
if not inventory:
return
for hash in inventory.Hashes:
hash = hash.encode('utf-8')
item = None
# try to get the inventory to send from relay cache
if hash in self.leader.RelayCache.keys():
item = self.leader.RelayCache[hash]
if inventory.Type == InventoryType.TXInt:
if not item:
item, index = BC.Default().GetTransaction(hash)
if not item:
item = self.leader.GetTransaction(hash)
if item:
message = Message(command='tx',
payload=item,
print_payload=False)
self.SendSerializedMessage(message)
elif inventory.Type == InventoryType.BlockInt:
if not item:
item = BC.Default().GetBlock(hash)
if item:
message = Message(command='block',
payload=item,
print_payload=False)
self.SendSerializedMessage(message)
elif inventory.Type == InventoryType.ConsensusInt:
if item:
self.SendSerializedMessage(
Message(command='consensus',
payload=item,
print_payload=False))
def test_data_received(self, mock):
node = NeoNode()
node.endpoint = Endpoint('hello.com', 1234)
node.host = node.endpoint.host
node.port = node.endpoint.port
payload = VersionPayload(10234, 1234, 'version')
message = Message('version', payload=payload)
stream = StreamManager.GetStream()
writer = BinaryWriter(stream)
message.Serialize(writer)
out = stream.getvalue()
node.dataReceived(out)
mock.assert_called_once()
self.assertEqual(node.Version.Nonce, payload.Nonce)
def HandleVerack(self):
"""Handle the `verack` response."""
m = Message('verack')
self.SendSerializedMessage(m)
self.leader.NodeCount += 1
self.identifier = self.leader.NodeCount
logger.debug(f"{self.prefix} Handshake complete!")
self.handshake_complete = True
self.ProtocolReady()
def AskForMoreHeaders(self):
logger.debug(
f"{self.prefix} asking for more headers, starting from {BC.Default().HeaderHeight}"
)
self.health_check(HEARTBEAT_HEADERS)
get_headers_message = Message(
"getheaders",
GetBlocksPayload(hash_start=[BC.Default().CurrentHeaderHash]))
self.SendSerializedMessage(get_headers_message)
def test_handle_message(self, mock):
node = NeoNode()
node.endpoint = Endpoint('hello.com', 1234)
node.host = node.endpoint.host
node.port = node.endpoint.port
payload = VersionPayload(10234, 1234, 'version')
message = Message('version', payload=payload)
stream = StreamManager.GetStream()
writer = BinaryWriter(stream)
message.Serialize(writer)
out = stream.getvalue()
print("OUT %s " % out)
out1 = out[0:10]
out2 = out[10:20]
out3 = out[20:]
node.dataReceived(out1)
node.dataReceived(out2)
self.assertEqual(node.buffer_in, out1 + out2)
# import pdb
# pdb.set_trace()
self.assertEqual(node.bytes_in, 20)
mock.assert_not_called()
node.dataReceived(out3)
self.assertEqual(node.bytes_in, len(out))
# mock.assert_called_with(message)
mock.assert_called_once()
def CheckDataReceived(self):
if len(self.buffer_in) >= 24:
mstart = self.buffer_in[:24]
ms = StreamManager.GetStream(mstart)
reader = BinaryReader(ms)
try:
m = Message()
m.Magic = reader.ReadUInt32()
m.Command = reader.ReadFixedString(12).decode('utf-8')
m.Length = reader.ReadUInt32()
m.Checksum = reader.ReadUInt32()
self.pm = m
except Exception as e:
self.Log('could not read initial bytes %s ' % e)
finally:
StreamManager.ReleaseStream(ms)
del reader
try:
#make this threadsafe
# reactor.callFromThread(self.CheckMessageData)
self.CheckMessageData()
except RecursionError:
self.Log("Recursion error!!!")
self.Disconnect()
def SendPeerInfo(self):
if not self.leader.ServiceEnabled:
return
peerlist = []
for peer in self.leader.Peers:
addr = peer.GetNetworkAddressWithTime()
if addr is not None:
peerlist.append(addr)
self.Log("Peer list %s " % list(map(lambda p: p.ToString(), peerlist)))
addrpayload = AddrPayload(addresses=peerlist)
message = Message('addr', addrpayload)
self.SendSerializedMessage(message)
def SendPeerInfo(self):
self.Log("SENDING PEER INFO %s " % self)
peerlist = []
for peer in self.leader.Peers:
#make sure peers rating is <= 90 by checking the banlist
if (peer.getRating <= 90):
peerlist.append(peer.GetNetworkAddressWithTime())
self.Log("Peer list %s " % peerlist)
addrpayload = AddrPayload(addresses=peerlist)
message = Message('addr', addrpayload)
self.SendSerializedMessage(message)
def SendPeerInfo(self):
# if not self.leader.ServiceEnabled:
# return
peerlist = []
for peer in self.leader.Peers:
addr = peer.GetNetworkAddressWithTime()
if addr is not None:
peerlist.append(addr)
peer_str_list = list(map(lambda p: p.ToString(), peerlist))
logger.debug(f"{self.prefix} Sending Peer list {peer_str_list}")
addrpayload = AddrPayload(addresses=peerlist)
message = Message('addr', addrpayload)
self.SendSerializedMessage(message)
def Relay(self, inventory):
"""
Wrap the inventory in a InvPayload object and send it over the write to the remote node.
Args:
inventory:
Returns:
bool: True (fixed)
"""
inventory = InvPayload(type=inventory.InventoryType, hashes=[inventory.Hash.ToBytes()])
m = Message("inv", inventory)
self.SendSerializedMessage(m)
return True
def CheckMessageData(self):
if not self.pm: return
currentlength = len(self.buffer_in)
messageExpectedLength = 24 + self.pm.Length
# percentcomplete = int(100 * (currentlength / messageExpectedLength))
# self.Log("Receiving %s data: %s percent complete" % (self.pm.Command, percentcomplete))
if currentlength >= messageExpectedLength:
mdata = self.buffer_in[:messageExpectedLength]
stream = StreamManager.GetStream(mdata)
reader = BinaryReader(stream)
message = Message()
message.Deserialize(reader)
StreamManager.ReleaseStream(stream)
self.buffer_in = self.buffer_in[messageExpectedLength:]
self.pm = None
self.MessageReceived(message)
self.reset_counter = False
while len(self.buffer_in) > 24 and not self.reset_counter:
self.CheckDataReceived()
else:
self.reset_counter = True
def DoAskForMoreBlocks(self):
hashes = []
hashstart = BC.Default().Height + 1
current_header_height = BC.Default().HeaderHeight + 1
do_go_ahead = False
if BC.Default().BlockSearchTries > 100 and len(
BC.Default().BlockRequests) > 0:
do_go_ahead = True
first = None
while hashstart <= current_header_height and len(
hashes) < self.leader.BREQPART:
hash = BC.Default().GetHeaderHash(hashstart)
if not do_go_ahead:
if hash is not None and hash not in BC.Default().BlockRequests \
and hash not in self.myblockrequests:
if not first:
first = hashstart
BC.Default().BlockRequests.add(hash)
self.myblockrequests.add(hash)
hashes.append(hash)
else:
if hash is not None:
if not first:
first = hashstart
BC.Default().BlockRequests.add(hash)
self.myblockrequests.add(hash)
hashes.append(hash)
hashstart += 1
if len(hashes) > 0:
logger.debug(
f"{self.prefix} asking for more blocks {first} - {hashstart} ({len(hashes)}) stale count: {BC.Default().BlockSearchTries} "
f"BCRLen: {len(BC.Default().BlockRequests)}")
self.health_check(HEARTBEAT_BLOCKS)
message = Message("getdata", InvPayload(InventoryType.Block,
hashes))
self.SendSerializedMessage(message)
def test_message_serialization(self):
message = Message('version', payload=self.payload)
self.assertEqual(message.Command, 'version')
ms = StreamManager.GetStream()
writer = BinaryWriter(ms)
message.Serialize(writer)
result = binascii.unhexlify( ms.ToArray())
StreamManager.ReleaseStream(ms)
ms = StreamManager.GetStream(result)
reader = BinaryReader(ms)
deserialized_message = Message()
deserialized_message.Deserialize( reader )
StreamManager.ReleaseStream(ms)
dm = deserialized_message
self.assertEqual(dm.Command, 'version')
self.assertEqual(dm.Magic, Settings.MAGIC)
checksum = Message.GetChecksum(dm.Payload)
self.assertEqual(checksum, dm.Checksum)
deserialized_version = IOHelper.AsSerializableWithType(dm.Payload, 'neo.Network.Payloads.VersionPayload.VersionPayload')
self.assertEqual(deserialized_version.Port, self.port)
self.assertEqual(deserialized_version.UserAgent, self.ua)
self.assertEqual(deserialized_version.Timestamp, self.payload.Timestamp)
def AskForMoreBlocks(self):
hashes = []
hashstart = BC.Default().Height + 1
self.Log("asking for more blocks ... %s " % hashstart)
while hashstart < BC.Default().HeaderHeight and len(hashes) < self.leader.BREQPART:
hash = BC.Default().GetHeaderHash(hashstart)
if not hash in BC.Default().BlockRequests and not hash in self.myblockrequests:
BC.Default().BlockRequests.add(hash)
self.myblockrequests.add(hash)
hashes.append(hash)
hashstart += 1
if len(hashes) > 0:
message = Message("getdata", InvPayload(InventoryType.Block, hashes))
self.SendSerializedMessage(message)
else:
reactor.callLater(5, self.AskForMoreBlocks)
def CheckDataReceived(self):
"""Tries to extract a Message from the data buffer and process it."""
currentLength = len(self.buffer_in)
if currentLength < 24:
return
# Extract the message header from the buffer, and return if not enough
# buffer to fully deserialize the message object.
try:
# Construct message
mstart = self.buffer_in[:24]
ms = StreamManager.GetStream(mstart)
reader = BinaryReader(ms)
m = Message()
# Extract message metadata
m.Magic = reader.ReadUInt32()
m.Command = reader.ReadFixedString(12).decode('utf-8')
m.Length = reader.ReadUInt32()
m.Checksum = reader.ReadUInt32()
# Return if not enough buffer to fully deserialize object.
messageExpectedLength = 24 + m.Length
# percentcomplete = int(100 * (currentLength / messageExpectedLength))
# self.Log("Receiving %s data: %s percent complete" % (m.Command, percentcomplete))
if currentLength < messageExpectedLength:
return
except Exception as e:
self.Log('Error: Could not read initial bytes %s ' % e)
return
finally:
StreamManager.ReleaseStream(ms)
del reader
# The message header was successfully extracted, and we have enough enough buffer
# to extract the full payload
try:
# Extract message bytes from buffer and truncate buffer
mdata = self.buffer_in[:messageExpectedLength]
self.buffer_in = self.buffer_in[messageExpectedLength:]
# Deserialize message with payload
stream = StreamManager.GetStream(mdata)
reader = BinaryReader(stream)
message = Message()
message.Deserialize(reader)
# Propagate new message
self.MessageReceived(message)
except Exception as e:
self.Log('Error: Could not extract message: %s ' % e)
return
finally:
StreamManager.ReleaseStream(stream)
# Finally, after a message has been fully deserialized and propagated,
# check if another message can be extracted with the current buffer:
if len(self.buffer_in) >= 24:
self.CheckDataReceived()
def CheckDataReceived(self):
"""Tries to extract a Message from the data buffer and process it."""
currentLength = len(self.buffer_in)
if currentLength < 24:
return
# Extract the message header from the buffer, and return if not enough
# buffer to fully deserialize the message object.
try:
# Construct message
mstart = self.buffer_in[:24]
ms = StreamManager.GetStream(mstart)
reader = BinaryReader(ms)
m = Message()
# Extract message metadata
m.Magic = reader.ReadUInt32()
m.Command = reader.ReadFixedString(12).decode('utf-8')
m.Length = reader.ReadUInt32()
m.Checksum = reader.ReadUInt32()
# Return if not enough buffer to fully deserialize object.
messageExpectedLength = 24 + m.Length
if currentLength < messageExpectedLength:
return
except Exception as e:
self.Log('Error: Could not read initial bytes %s ' % e)
return
finally:
StreamManager.ReleaseStream(ms)
del reader
# The message header was successfully extracted, and we have enough enough buffer
# to extract the full payload
try:
# Extract message bytes from buffer and truncate buffer
mdata = self.buffer_in[:messageExpectedLength]
self.buffer_in = self.buffer_in[messageExpectedLength:]
# Deserialize message with payload
stream = StreamManager.GetStream(mdata)
reader = BinaryReader(stream)
message = Message()
message.Deserialize(reader)
# Propagate new message
self.MessageReceived(message)
except Exception as e:
self.Log('Error: Could not extract message: %s ' % e)
return
finally:
StreamManager.ReleaseStream(stream)
# Finally, after a message has been fully deserialized and propagated,
# check if another message can be extracted with the current buffer:
if len(self.buffer_in) >= 24:
self.CheckDataReceived()
def RequestVersion(self):
"""Request the remote client version."""
# self.Log("All caught up, requesting version")
m = Message("getversion")
self.SendSerializedMessage(m)
def HandleVerack(self):
"""Handle the `verack` response."""
m = Message('verack')
self.SendSerializedMessage(m)
self.ProtocolReady()
def RequestVersion(self):
"""Request the remote client version."""
m = Message("getversion")
self.SendSerializedMessage(m)
def RequestPeerInfo(self):
"""Request the peer address information from the remote client."""
self.SendSerializedMessage(Message('getaddr'))
def DoAskForSingleBlock(self, block_hash):
if block_hash not in self.myblockrequests:
message = Message("getdata", InvPayload(InventoryType.Block, [block_hash]))
self.myblockrequests.add(block_hash)
self.SendSerializedMessage(message)
def AskForMoreHeaders(self):
# self.Log("asking for more headers...")
get_headers_message = Message("getheaders", GetBlocksPayload(hash_start=[BC.Default().CurrentHeaderHash]))
self.SendSerializedMessage(get_headers_message)