def __getitem__(self, key):
stored_block = self._block_store[key]
if stored_block is not None:
block = Block()
block.ParseFromString(stored_block)
return BlockWrapper(status=BlockStatus.Valid, block=block)
raise KeyError("Key {} not found.".format(key))
def handle(self, connection_id, message_content):
gossip_message = network_pb2.GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == network_pb2.GossipMessage.BLOCK:
block = Block()
block.ParseFromString(gossip_message.content)
LOGGER.debug("CompleterGossipHandler: BLOCK=%s",
block.header_signature[:8])
self._completer.add_block(block)
elif gossip_message.content_type == network_pb2.GossipMessage.BATCH:
batch = Batch()
batch.ParseFromString(gossip_message.content)
#candidate_id = gossip_message.candidate_id.hex()
LOGGER.debug("CompleterGossipHandler: NEW BATCH=%s ",
batch.header_signature[:8])
# works in case batch from another node
self._completer.add_batch(batch)
elif gossip_message.content_type == network_pb2.GossipMessage.BATCHES:
batches = BatchList()
batches.ParseFromString(gossip_message.content)
candidate_id = batches.candidate_id.hex()
num = len(batches.batches)
#LOGGER.debug("CompleterGossipHandler: NEW BATCHES=%s candidate_id=%s", batches,candidate_id[:8])
block_num = batches.block_num
for batch in batches.batches:
LOGGER.debug(" => NEW BATCH[%s]=%s candidate_id=%s.%s", num,
batch.header_signature[:8], block_num,
candidate_id[:8])
self._completer.add_batch(batch,
(candidate_id, block_num, num))
return HandlerResult(status=HandlerStatus.PASS)
def handle(self, connection_id, message_content):
exclude = [connection_id]
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.time_to_live == 0:
# Do not forward message if it has reached its time to live limit
return HandlerResult(status=HandlerStatus.PASS)
else:
# decrement time_to_live
time_to_live = gossip_message.time_to_live
gossip_message.time_to_live = time_to_live - 1
if gossip_message.content_type == GossipMessage.BATCH:
batch = Batch()
batch.ParseFromString(gossip_message.content)
# If we already have this batch, don't forward it
if not self._completer.get_batch(batch.header_signature):
self._gossip.broadcast_batch(batch, exclude)
elif gossip_message.content_type == GossipMessage.BLOCK:
block = Block()
block.ParseFromString(gossip_message.content)
# If we already have this block, don't forward it
if not self._completer.get_block(block.header_signature):
self._gossip.broadcast_block(block, exclude)
else:
LOGGER.info("received %s, not BATCH or BLOCK",
gossip_message.content_type)
return HandlerResult(status=HandlerStatus.PASS)
def handle(self, connection_id, message_content):
message = GossipMessage()
message.ParseFromString(message_content)
if message.content_type == "BLOCK":
public_key = \
self._network.connection_id_to_public_key(connection_id)
block = Block()
block.ParseFromString(message.content)
header = BlockHeader()
header.ParseFromString(block.header)
if header.signer_public_key == public_key:
permitted = \
self._permission_verifier.check_network_consensus_role(
public_key)
if not permitted:
LOGGER.debug(
"Public key is not permitted to publish block, "
"remove connection: %s", connection_id)
self._gossip.unregister_peer(connection_id)
violation = AuthorizationViolation(
violation=RoleType.Value("NETWORK"))
return HandlerResult(HandlerStatus.RETURN_AND_CLOSE,
message_out=violation,
message_type=validator_pb2.Message.
AUTHORIZATION_VIOLATION)
# if allowed pass message
return HandlerResult(HandlerStatus.PASS)
def forks(self, head):
(vec_ptr, vec_len,
vec_cap) = ffi.prepare_vec_result(pointer_type=_BlockPayload)
head = ctypes.c_char_p(head.encode())
_libexec('chain_controller_forks', self.pointer, head,
ctypes.byref(vec_ptr), ctypes.byref(vec_len),
ctypes.byref(vec_cap))
# Check if NULL
if not vec_ptr:
return None
blocks = []
for i in range(vec_len.value):
block_payload = vec_ptr[i]
payload = ffi.from_rust_vec(
block_payload.block_ptr,
ctypes.c_size_t(block_payload.block_len),
ctypes.c_size_t(block_payload.block_cap),
)
block = Block()
block.ParseFromString(payload)
blocks.append(BlockWrapper(block))
LIBRARY.call("chain_controller_reclaim_block_payload_vec", vec_ptr,
vec_len, vec_cap)
return blocks
def test_block_cache(self):
block_store = {}
cache = BlockCache(block_store=block_store, keep_time=1,
purge_frequency=1)
header1 = BlockHeader(previous_block_id="000")
block1 = BlockWrapper(Block(header=header1.SerializeToString(),
header_signature="ABC"))
header2 = BlockHeader(previous_block_id="ABC")
block2 = BlockWrapper(Block(header=header2.SerializeToString(),
header_signature="DEF"))
header3 = BlockHeader(previous_block_id="BCA")
block3 = BlockWrapper(Block(header=header3.SerializeToString(),
header_signature="FED"))
cache[block1.header_signature] = block1
cache[block2.header_signature] = block2
# Check that blocks are in the BlockCache
self.assertIn("ABC", cache)
self.assertIn("DEF", cache)
# Wait for purge time to expire
time.sleep(1)
# Add "FED"
cache[block3.header_signature] = block3
# Check that "ABC" is still in the cache even though the keep time has
# expired because it has a referecne count of 1 but "DEF" has been
# removed
self.assertIn("ABC", cache)
self.assertNotIn("DEF", cache)
self.assertIn("FED", cache)
def handle(self, connection_id, message_content):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == gossip_message.BLOCK:
block = Block()
block.ParseFromString(gossip_message.content)
has_block = False
if self._completer.get_block(block.header_signature) is not None:
has_block = True
if not has_block and self._has_block(block.header_signature):
has_block = True
if has_block:
LOGGER.debug("Drop duplicate block: %s",
block.header_signature)
return HandlerResult(HandlerStatus.DROP)
if gossip_message.content_type == gossip_message.BATCH:
batch = Batch()
batch.ParseFromString(gossip_message.content)
has_batch = False
if self._completer.get_batch(batch.header_signature) is not None:
has_batch = True
if not has_batch and self._has_batch(batch.header_signature):
has_batch = True
if has_batch:
LOGGER.debug("Drop duplicate batch: %s",
batch.header_signature)
return HandlerResult(HandlerStatus.DROP)
return HandlerResult(HandlerStatus.PASS)
def handle(self, connection_id, message_content):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == "BLOCK":
block = Block()
block.ParseFromString(gossip_message.content)
status = validate_block(block)
if status is True:
LOGGER.debug("block passes signature verification %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.PASS)
LOGGER.debug("block signature is invalid: %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
elif gossip_message.content_type == "BATCH":
batch = Batch()
batch.ParseFromString(gossip_message.content)
status = validate_batch(batch)
if status is True:
LOGGER.debug("batch passes signature verification %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.PASS)
LOGGER.debug("batch signature is invalid: %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
def handle(self, connection_id, message_content):
gossip_message = network_pb2.GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == "BLOCK":
block = Block()
block.ParseFromString(gossip_message.content)
if not is_valid_block(block):
LOGGER.debug("block's batches structure is invalid: %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
LOGGER.debug("block passes batch structure verification %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.PASS)
elif gossip_message.content_type == "BATCH":
batch = Batch()
batch.ParseFromString(gossip_message.content)
if not is_valid_batch(batch):
LOGGER.debug("batch structure is invalid: %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
LOGGER.debug("batch passes structure verification %s",
batch.header_signature)
return HandlerResult(status=HandlerStatus.PASS)
return HandlerResult(status=HandlerStatus.PASS)
def handle(self, connection_id, message_content):
block_response_message = network_pb2.GossipBlockResponse()
block_response_message.ParseFromString(message_content)
block = Block()
block.ParseFromString(block_response_message.content)
self._completer.add_block(block)
return HandlerResult(status=HandlerStatus.PASS)
def gossip_block_response_preprocessor(message_content_bytes):
block_response = GossipBlockResponse()
block_response.ParseFromString(message_content_bytes)
block = Block()
block.ParseFromString(block_response.content)
content = block, message_content_bytes
return PreprocessorResult(content=content)
def __getitem__(self, key):
stored_block = self._block_store[key]
# Block id strings are stored under batch/txn ids for reference.
# Only Blocks, not ids or Nones, should be returned by __getitem__.
if isinstance(stored_block, bytes):
block = Block()
block.ParseFromString(stored_block)
return BlockWrapper(status=BlockStatus.Valid, block=block)
raise KeyError('Block "{}" not found in store'.format(key))
def test_no_block(self):
"""
Test that if the block does not exist yet in the completer or the
chain controller, the gossip message is passed.
"""
block = Block(header_signature="Block1")
message = GossipMessage(content_type=GossipMessage.BLOCK,
content=block.SerializeToString())
handler_status = self.handler.handle("connection_id",
message.SerializeToString())
self.assertEqual(handler_status.status, HandlerStatus.PASS)
def handle(self, connection_id, message_content):
block_response_message = network_pb2.GossipBlockResponse()
block_response_message.ParseFromString(message_content)
block = Block()
block.ParseFromString(block_response_message.content)
LOGGER.debug("CompleterGossipBlockResponseHandler: BLOCK=%s nest=%s",
block.header_signature[:8], block_response_message.nest)
self._completer.add_block(block, nest=block_response_message.nest)
return HandlerResult(status=HandlerStatus.PASS)
def run(self):
while True:
if self._exit:
return
try:
request = self.incoming_msg_queue.get(block=False)
if request.content_type == "Block":
try:
block = Block()
block.ParseFromString(request.content)
status = self.validate_block(block)
if status:
self.outgoing_msg_queue.put_nowait(request)
with self.out_condition:
self.out_condition.notify_all()
LOGGER.info("Block signature is invalid: %s",
block.header_signature)
except DecodeError as e:
# what to do with a bad msg
LOGGER.warning(
"Problem decoding GossipMessage for "
"Block, %s", e)
elif request.content_type == "Batch":
try:
batch = Batch()
batch.ParseFromString(request.content)
status = self.validate_batch(batch)
if status:
self.outgoing_msg_queue.put_nowait(request)
with self.out_condition:
self.out_condition.notify_all()
else:
LOGGER.info("Batch signature is invalid: %s",
batch.header_signature)
except DecodeError as e:
LOGGER.warning(
"Problem decoding GossipMessage for "
"Batch, %s", e)
elif request.content_type == "BlockRequest":
self.outgoing_msg_queue.put_notwait(request)
with self.out_condition:
self.out_condition.notify_all()
elif request.content_type == "Test":
LOGGER.debug("Verifier Handle Test")
self.outgoing_msg_queue.put_nowait(request)
with self.out_condition:
self.out_condition.notify_all()
except queue.Empty:
with self.in_condition:
self.in_condition.wait()
def handle(self, connection_id, message_content):
gossip_message = network_pb2.GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == network_pb2.GossipMessage.BLOCK:
block = Block()
block.ParseFromString(gossip_message.content)
self._completer.add_block(block)
elif gossip_message.content_type == network_pb2.GossipMessage.BATCH:
batch = Batch()
batch.ParseFromString(gossip_message.content)
self._completer.add_batch(batch)
return HandlerResult(status=HandlerStatus.PASS)
def handle(self, connection_id, message_content):
block_response_message = GossipBlockResponse()
block_response_message.ParseFromString(message_content)
block = Block()
block.ParseFromString(block_response_message.content)
if not is_valid_block(block):
LOGGER.debug("requested block's batches structure is invalid: %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
return HandlerResult(status=HandlerStatus.PASS)
def _remove_batches(self, block, keep_batches):
"""Returns a copy of the block without the non-injected
batches, which other validators are likely to already
have. This reduces the size of the broadcasted block.
"""
clone = Block()
clone.header = block.header
clone.header_signature = block.header_signature
# tupple is faster than list
clone.batches.extend((batch for batch in block.batches
if batch.header_signature in keep_batches))
return clone
def do_load_from_block_store():
bs = {}
block1 = Block(
header=BlockHeader(previous_block_id="000").SerializeToString(),
header_signature="test")
bs["test"] = BlockWrapper(block1)
block2 = Block(
header=BlockHeader(previous_block_id="000").SerializeToString(),
header_signature="test2")
blkw2 = BlockWrapper(block2)
bs["test2"] = blkw2
bc = BlockCache(bs)
return bc, blkw2
def _remove_batches(self, block, keep_batches):
"""Returns a copy of the block without the non-injected
batches, which other validators are likely to already
have. This reduces the size of the broadcasted block.
"""
clone = Block()
clone.header = block.header
clone.header_signature = block.header_signature
clone.batches.extend([
batch for batch in block.batches
if batch.header_signature in keep_batches
])
return clone
def run(self):
if self.incoming_msg_queue is None or self.condition is None:
LOGGER.warning("Dispatcher can not be used if incoming_msg_queue "
"or condition is not set.")
return
while True:
if self._exit:
return
try:
request = self.incoming_msg_queue.get(block=False)
if request.content_type == "Block":
try:
block = Block()
block.ParseFromString(request.content)
LOGGER.debug("Block given to Completer %s",
block.header_signature)
self.completer_queue.put_nowait(block)
with self.completer_conditon:
self.completer_conditon.notify_all()
except DecodeError as e:
# what to do with a bad msg
LOGGER.warning(
"Problem decoding GossipMessage for "
"Block, %s", e)
elif request.content_type == "Batch":
try:
batch = Batch()
batch.ParseFromString(request.content)
self.completer.add_batch(batch)
self.on_batch_received(batch)
except DecodeError as e:
LOGGER.warning(
"Problem decoding GossipMessage for "
"Batch, %s", e)
elif request.content_type == "BlockRequest":
block_id = str(request.content, "utf-8")
self.on_block_requested(block_id)
elif request.content_type == "Test":
LOGGER.debug("Dispatch Handle Test")
except queue.Empty:
with self.condition:
self.condition.wait()
def deserialize_block(value):
"""
Deserialize a byte string into a BlockWrapper
Args:
value (bytes): the byte string to deserialze
Returns:
BlockWrapper: a block wrapper instance
"""
# Block id strings are stored under batch/txn ids for reference.
# Only Blocks, not ids or Nones, should be returned by _get_block.
block = Block()
block.ParseFromString(value)
return BlockWrapper(status=BlockStatus.Valid, block=block)
def handle(self, identity, message_content):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == "BATCH":
batch = Batch()
batch.ParseFromString(gossip_message.content)
self._gossip.broadcast_batch(batch)
elif gossip_message.content_type == "BLOCK":
block = Block()
block.ParseFromString(gossip_message.content)
self._gossip.broadcast_block(block)
else:
LOGGER.info("received %s, not BATCH or BLOCK",
gossip_message.content_type)
return HandlerResult(status=HandlerStatus.PASS)
def handle(self, connection_id, message_content):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content)
if gossip_message.content_type == GossipMessage.BLOCK:
block = Block()
block.ParseFromString(gossip_message.content)
if block.header_signature in self._seen_cache:
LOGGER.debug("Drop already validated block: %s cache=%s",
block.header_signature[:8], len(self._seen_cache)
) #[sig[:8] for sig in self._seen_cache])
return HandlerResult(status=HandlerStatus.DROP)
if not is_valid_block(block):
LOGGER.debug("block signature is invalid: %s",
block.header_signature[:8])
return HandlerResult(status=HandlerStatus.DROP)
self._seen_cache[block.header_signature] = None
return HandlerResult(status=HandlerStatus.PASS)
elif gossip_message.content_type == GossipMessage.BATCH:
batch = Batch()
batch.ParseFromString(gossip_message.content)
if batch.header_signature in self._seen_cache:
LOGGER.debug("Drop already validated batch: %s",
batch.header_signature[:8])
return HandlerResult(status=HandlerStatus.DROP)
if not is_valid_batch(batch):
LOGGER.debug("batch signature is invalid: %s",
batch.header_signature[:8])
return HandlerResult(status=HandlerStatus.DROP)
self._seen_cache[batch.header_signature] = None
return HandlerResult(status=HandlerStatus.PASS)
elif gossip_message.content_type == GossipMessage.BATCHES:
# check batches signature
LOGGER.debug("Check BATCHES signature!!!")
return HandlerResult(status=HandlerStatus.PASS)
elif gossip_message.content_type == GossipMessage.ENDPOINTS:
# check endpoints
LOGGER.debug("Check ENDPOINTS signature!!!")
return HandlerResult(status=HandlerStatus.PASS)
# should drop the message if it does not have a valid content_type
return HandlerResult(status=HandlerStatus.DROP)
def handle(self, connection_id, message_content):
block_response_message = GossipBlockResponse()
block_response_message.ParseFromString(message_content)
block = Block()
block.ParseFromString(block_response_message.content)
status = validate_block(block)
if status is True:
LOGGER.debug("requested block passes signature verification %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.PASS)
else:
LOGGER.debug("requested block's signature is invalid: %s",
block.header_signature)
return HandlerResult(status=HandlerStatus.DROP)
def _create_block(self):
return BlockWrapper.wrap(
Block(header_signature='some_block_id',
batches=[],
header=BlockHeader(block_num=0,
previous_block_id=NULL_BLOCK_IDENTIFIER).
SerializeToString()))
def gen_block(self, block_id, prev_id, num, batches):
return BlockWrapper(
Block(header_signature=block_id,
batches=batches,
header=BlockHeader(
block_num=num,
previous_block_id=prev_id).SerializeToString()))
def add_block(self, base_id, root=b'merkle_root'.hex()):
block_id = 'b' * (128 - len(base_id)) + base_id
head = self.chain_head
if head:
previous_id = head.header_signature
num = head.header.block_num + 1
else:
previous_id = 'zzzzz'
num = 0
signer_public_key = b'public_key' + bytes(base_id, 'utf-8')
header = BlockHeader(
block_num=num,
previous_block_id=previous_id,
signer_public_key=signer_public_key.hex(),
batch_ids=[block_id],
consensus=b'consensus',
state_root_hash=root)
block = Block(
header=header.SerializeToString(),
header_signature=block_id,
batches=[make_mock_batch(base_id)])
self.put_blocks([block])
def gossip_message_preprocessor(message_content_bytes):
gossip_message = GossipMessage()
gossip_message.ParseFromString(message_content_bytes)
tag = gossip_message.content_type
if tag == GossipMessage.BLOCK:
obj = Block()
obj.ParseFromString(gossip_message.content)
elif tag == GossipMessage.BATCH:
obj = Batch()
obj.ParseFromString(gossip_message.content)
content = obj, tag, gossip_message.time_to_live
return PreprocessorResult(content=content)
def chain_head_fn(self):
(vec_ptr, vec_len, vec_cap) = ffi.prepare_vec_result()
_libexec('chain_controller_chain_head', self.pointer,
ctypes.byref(vec_ptr), ctypes.byref(vec_len),
ctypes.byref(vec_cap))
# Check if NULL
if not vec_ptr:
return None
payload = ffi.from_rust_vec(vec_ptr, vec_len, vec_cap)
block = Block()
block.ParseFromString(payload)
return block
def _create_blocks(self,
block_count,
batch_count,
valid_block=True,
valid_batch=True):
block_list = []
for i in range(block_count):
batch_list = self._create_batches(batch_count,
2,
valid_batch=valid_batch)
batch_ids = [batch.header_signature for batch in batch_list]
block_header = BlockHeader(signer_pubkey=self.public_key,
batch_ids=batch_ids)
header_bytes = block_header.SerializeToString()
if valid_block:
signature = signing.sign(header_bytes, self.private_key)
else:
signature = "bad_signature"
block = Block(header=header_bytes,
batches=batch_list,
header_signature=signature)
block_list.append(block)
return block_list
