def __getitem__(self, key): stored_block = self._block_store[key] if stored_block is not None: block = Block() block.ParseFromString(stored_block) return BlockWrapper(status=BlockStatus.Valid, block=block) raise KeyError("Key {} not found.".format(key))
def handle(self, connection_id, message_content): gossip_message = network_pb2.GossipMessage() gossip_message.ParseFromString(message_content) if gossip_message.content_type == network_pb2.GossipMessage.BLOCK: block = Block() block.ParseFromString(gossip_message.content) LOGGER.debug("CompleterGossipHandler: BLOCK=%s", block.header_signature[:8]) self._completer.add_block(block) elif gossip_message.content_type == network_pb2.GossipMessage.BATCH: batch = Batch() batch.ParseFromString(gossip_message.content) #candidate_id = gossip_message.candidate_id.hex() LOGGER.debug("CompleterGossipHandler: NEW BATCH=%s ", batch.header_signature[:8]) # works in case batch from another node self._completer.add_batch(batch) elif gossip_message.content_type == network_pb2.GossipMessage.BATCHES: batches = BatchList() batches.ParseFromString(gossip_message.content) candidate_id = batches.candidate_id.hex() num = len(batches.batches) #LOGGER.debug("CompleterGossipHandler: NEW BATCHES=%s candidate_id=%s", batches,candidate_id[:8]) block_num = batches.block_num for batch in batches.batches: LOGGER.debug(" => NEW BATCH[%s]=%s candidate_id=%s.%s", num, batch.header_signature[:8], block_num, candidate_id[:8]) self._completer.add_batch(batch, (candidate_id, block_num, num)) return HandlerResult(status=HandlerStatus.PASS)
def handle(self, connection_id, message_content): exclude = [connection_id] gossip_message = GossipMessage() gossip_message.ParseFromString(message_content) if gossip_message.time_to_live == 0: # Do not forward message if it has reached its time to live limit return HandlerResult(status=HandlerStatus.PASS) else: # decrement time_to_live time_to_live = gossip_message.time_to_live gossip_message.time_to_live = time_to_live - 1 if gossip_message.content_type == GossipMessage.BATCH: batch = Batch() batch.ParseFromString(gossip_message.content) # If we already have this batch, don't forward it if not self._completer.get_batch(batch.header_signature): self._gossip.broadcast_batch(batch, exclude) elif gossip_message.content_type == GossipMessage.BLOCK: block = Block() block.ParseFromString(gossip_message.content) # If we already have this block, don't forward it if not self._completer.get_block(block.header_signature): self._gossip.broadcast_block(block, exclude) else: LOGGER.info("received %s, not BATCH or BLOCK", gossip_message.content_type) return HandlerResult(status=HandlerStatus.PASS)
def handle(self, connection_id, message_content): message = GossipMessage() message.ParseFromString(message_content) if message.content_type == "BLOCK": public_key = \ self._network.connection_id_to_public_key(connection_id) block = Block() block.ParseFromString(message.content) header = BlockHeader() header.ParseFromString(block.header) if header.signer_public_key == public_key: permitted = \ self._permission_verifier.check_network_consensus_role( public_key) if not permitted: LOGGER.debug( "Public key is not permitted to publish block, " "remove connection: %s", connection_id) self._gossip.unregister_peer(connection_id) violation = AuthorizationViolation( violation=RoleType.Value("NETWORK")) return HandlerResult(HandlerStatus.RETURN_AND_CLOSE, message_out=violation, message_type=validator_pb2.Message. AUTHORIZATION_VIOLATION) # if allowed pass message return HandlerResult(HandlerStatus.PASS)
def forks(self, head): (vec_ptr, vec_len, vec_cap) = ffi.prepare_vec_result(pointer_type=_BlockPayload) head = ctypes.c_char_p(head.encode()) _libexec('chain_controller_forks', self.pointer, head, ctypes.byref(vec_ptr), ctypes.byref(vec_len), ctypes.byref(vec_cap)) # Check if NULL if not vec_ptr: return None blocks = [] for i in range(vec_len.value): block_payload = vec_ptr[i] payload = ffi.from_rust_vec( block_payload.block_ptr, ctypes.c_size_t(block_payload.block_len), ctypes.c_size_t(block_payload.block_cap), ) block = Block() block.ParseFromString(payload) blocks.append(BlockWrapper(block)) LIBRARY.call("chain_controller_reclaim_block_payload_vec", vec_ptr, vec_len, vec_cap) return blocks
def test_block_cache(self): block_store = {} cache = BlockCache(block_store=block_store, keep_time=1, purge_frequency=1) header1 = BlockHeader(previous_block_id="000") block1 = BlockWrapper(Block(header=header1.SerializeToString(), header_signature="ABC")) header2 = BlockHeader(previous_block_id="ABC") block2 = BlockWrapper(Block(header=header2.SerializeToString(), header_signature="DEF")) header3 = BlockHeader(previous_block_id="BCA") block3 = BlockWrapper(Block(header=header3.SerializeToString(), header_signature="FED")) cache[block1.header_signature] = block1 cache[block2.header_signature] = block2 # Check that blocks are in the BlockCache self.assertIn("ABC", cache) self.assertIn("DEF", cache) # Wait for purge time to expire time.sleep(1) # Add "FED" cache[block3.header_signature] = block3 # Check that "ABC" is still in the cache even though the keep time has # expired because it has a referecne count of 1 but "DEF" has been # removed self.assertIn("ABC", cache) self.assertNotIn("DEF", cache) self.assertIn("FED", cache)
def handle(self, connection_id, message_content): gossip_message = GossipMessage() gossip_message.ParseFromString(message_content) if gossip_message.content_type == gossip_message.BLOCK: block = Block() block.ParseFromString(gossip_message.content) has_block = False if self._completer.get_block(block.header_signature) is not None: has_block = True if not has_block and self._has_block(block.header_signature): has_block = True if has_block: LOGGER.debug("Drop duplicate block: %s", block.header_signature) return HandlerResult(HandlerStatus.DROP) if gossip_message.content_type == gossip_message.BATCH: batch = Batch() batch.ParseFromString(gossip_message.content) has_batch = False if self._completer.get_batch(batch.header_signature) is not None: has_batch = True if not has_batch and self._has_batch(batch.header_signature): has_batch = True if has_batch: LOGGER.debug("Drop duplicate batch: %s", batch.header_signature) return HandlerResult(HandlerStatus.DROP) return HandlerResult(HandlerStatus.PASS)
def handle(self, connection_id, message_content): gossip_message = GossipMessage() gossip_message.ParseFromString(message_content) if gossip_message.content_type == "BLOCK": block = Block() block.ParseFromString(gossip_message.content) status = validate_block(block) if status is True: LOGGER.debug("block passes signature verification %s", block.header_signature) return HandlerResult(status=HandlerStatus.PASS) LOGGER.debug("block signature is invalid: %s", block.header_signature) return HandlerResult(status=HandlerStatus.DROP) elif gossip_message.content_type == "BATCH": batch = Batch() batch.ParseFromString(gossip_message.content) status = validate_batch(batch) if status is True: LOGGER.debug("batch passes signature verification %s", batch.header_signature) return HandlerResult(status=HandlerStatus.PASS) LOGGER.debug("batch signature is invalid: %s", batch.header_signature) return HandlerResult(status=HandlerStatus.DROP)
def handle(self, connection_id, message_content): gossip_message = network_pb2.GossipMessage() gossip_message.ParseFromString(message_content) if gossip_message.content_type == "BLOCK": block = Block() block.ParseFromString(gossip_message.content) if not is_valid_block(block): LOGGER.debug("block's batches structure is invalid: %s", block.header_signature) return HandlerResult(status=HandlerStatus.DROP) LOGGER.debug("block passes batch structure verification %s", block.header_signature) return HandlerResult(status=HandlerStatus.PASS) elif gossip_message.content_type == "BATCH": batch = Batch() batch.ParseFromString(gossip_message.content) if not is_valid_batch(batch): LOGGER.debug("batch structure is invalid: %s", batch.header_signature) return HandlerResult(status=HandlerStatus.DROP) LOGGER.debug("batch passes structure verification %s", batch.header_signature) return HandlerResult(status=HandlerStatus.PASS) return HandlerResult(status=HandlerStatus.PASS)
def handle(self, connection_id, message_content): block_response_message = network_pb2.GossipBlockResponse() block_response_message.ParseFromString(message_content) block = Block() block.ParseFromString(block_response_message.content) self._completer.add_block(block) return HandlerResult(status=HandlerStatus.PASS)
def gossip_block_response_preprocessor(message_content_bytes): block_response = GossipBlockResponse() block_response.ParseFromString(message_content_bytes) block = Block() block.ParseFromString(block_response.content) content = block, message_content_bytes return PreprocessorResult(content=content)
def __getitem__(self, key): stored_block = self._block_store[key] # Block id strings are stored under batch/txn ids for reference. # Only Blocks, not ids or Nones, should be returned by __getitem__. if isinstance(stored_block, bytes): block = Block() block.ParseFromString(stored_block) return BlockWrapper(status=BlockStatus.Valid, block=block) raise KeyError('Block "{}" not found in store'.format(key))
def test_no_block(self): """ Test that if the block does not exist yet in the completer or the chain controller, the gossip message is passed. """ block = Block(header_signature="Block1") message = GossipMessage(content_type=GossipMessage.BLOCK, content=block.SerializeToString()) handler_status = self.handler.handle("connection_id", message.SerializeToString()) self.assertEqual(handler_status.status, HandlerStatus.PASS)
def handle(self, connection_id, message_content): block_response_message = network_pb2.GossipBlockResponse() block_response_message.ParseFromString(message_content) block = Block() block.ParseFromString(block_response_message.content) LOGGER.debug("CompleterGossipBlockResponseHandler: BLOCK=%s nest=%s", block.header_signature[:8], block_response_message.nest) self._completer.add_block(block, nest=block_response_message.nest) return HandlerResult(status=HandlerStatus.PASS)
def run(self): while True: if self._exit: return try: request = self.incoming_msg_queue.get(block=False) if request.content_type == "Block": try: block = Block() block.ParseFromString(request.content) status = self.validate_block(block) if status: self.outgoing_msg_queue.put_nowait(request) with self.out_condition: self.out_condition.notify_all() LOGGER.info("Block signature is invalid: %s", block.header_signature) except DecodeError as e: # what to do with a bad msg LOGGER.warning( "Problem decoding GossipMessage for " "Block, %s", e) elif request.content_type == "Batch": try: batch = Batch() batch.ParseFromString(request.content) status = self.validate_batch(batch) if status: self.outgoing_msg_queue.put_nowait(request) with self.out_condition: self.out_condition.notify_all() else: LOGGER.info("Batch signature is invalid: %s", batch.header_signature) except DecodeError as e: LOGGER.warning( "Problem decoding GossipMessage for " "Batch, %s", e) elif request.content_type == "BlockRequest": self.outgoing_msg_queue.put_notwait(request) with self.out_condition: self.out_condition.notify_all() elif request.content_type == "Test": LOGGER.debug("Verifier Handle Test") self.outgoing_msg_queue.put_nowait(request) with self.out_condition: self.out_condition.notify_all() except queue.Empty: with self.in_condition: self.in_condition.wait()
def handle(self, connection_id, message_content): gossip_message = network_pb2.GossipMessage() gossip_message.ParseFromString(message_content) if gossip_message.content_type == network_pb2.GossipMessage.BLOCK: block = Block() block.ParseFromString(gossip_message.content) self._completer.add_block(block) elif gossip_message.content_type == network_pb2.GossipMessage.BATCH: batch = Batch() batch.ParseFromString(gossip_message.content) self._completer.add_batch(batch) return HandlerResult(status=HandlerStatus.PASS)
def handle(self, connection_id, message_content): block_response_message = GossipBlockResponse() block_response_message.ParseFromString(message_content) block = Block() block.ParseFromString(block_response_message.content) if not is_valid_block(block): LOGGER.debug("requested block's batches structure is invalid: %s", block.header_signature) return HandlerResult(status=HandlerStatus.DROP) return HandlerResult(status=HandlerStatus.PASS)
def _remove_batches(self, block, keep_batches): """Returns a copy of the block without the non-injected batches, which other validators are likely to already have. This reduces the size of the broadcasted block. """ clone = Block() clone.header = block.header clone.header_signature = block.header_signature # tupple is faster than list clone.batches.extend((batch for batch in block.batches if batch.header_signature in keep_batches)) return clone
def do_load_from_block_store(): bs = {} block1 = Block( header=BlockHeader(previous_block_id="000").SerializeToString(), header_signature="test") bs["test"] = BlockWrapper(block1) block2 = Block( header=BlockHeader(previous_block_id="000").SerializeToString(), header_signature="test2") blkw2 = BlockWrapper(block2) bs["test2"] = blkw2 bc = BlockCache(bs) return bc, blkw2
def _remove_batches(self, block, keep_batches): """Returns a copy of the block without the non-injected batches, which other validators are likely to already have. This reduces the size of the broadcasted block. """ clone = Block() clone.header = block.header clone.header_signature = block.header_signature clone.batches.extend([ batch for batch in block.batches if batch.header_signature in keep_batches ]) return clone
def run(self): if self.incoming_msg_queue is None or self.condition is None: LOGGER.warning("Dispatcher can not be used if incoming_msg_queue " "or condition is not set.") return while True: if self._exit: return try: request = self.incoming_msg_queue.get(block=False) if request.content_type == "Block": try: block = Block() block.ParseFromString(request.content) LOGGER.debug("Block given to Completer %s", block.header_signature) self.completer_queue.put_nowait(block) with self.completer_conditon: self.completer_conditon.notify_all() except DecodeError as e: # what to do with a bad msg LOGGER.warning( "Problem decoding GossipMessage for " "Block, %s", e) elif request.content_type == "Batch": try: batch = Batch() batch.ParseFromString(request.content) self.completer.add_batch(batch) self.on_batch_received(batch) except DecodeError as e: LOGGER.warning( "Problem decoding GossipMessage for " "Batch, %s", e) elif request.content_type == "BlockRequest": block_id = str(request.content, "utf-8") self.on_block_requested(block_id) elif request.content_type == "Test": LOGGER.debug("Dispatch Handle Test") except queue.Empty: with self.condition: self.condition.wait()
def deserialize_block(value): """ Deserialize a byte string into a BlockWrapper Args: value (bytes): the byte string to deserialze Returns: BlockWrapper: a block wrapper instance """ # Block id strings are stored under batch/txn ids for reference. # Only Blocks, not ids or Nones, should be returned by _get_block. block = Block() block.ParseFromString(value) return BlockWrapper(status=BlockStatus.Valid, block=block)
def handle(self, identity, message_content): gossip_message = GossipMessage() gossip_message.ParseFromString(message_content) if gossip_message.content_type == "BATCH": batch = Batch() batch.ParseFromString(gossip_message.content) self._gossip.broadcast_batch(batch) elif gossip_message.content_type == "BLOCK": block = Block() block.ParseFromString(gossip_message.content) self._gossip.broadcast_block(block) else: LOGGER.info("received %s, not BATCH or BLOCK", gossip_message.content_type) return HandlerResult(status=HandlerStatus.PASS)
def handle(self, connection_id, message_content): gossip_message = GossipMessage() gossip_message.ParseFromString(message_content) if gossip_message.content_type == GossipMessage.BLOCK: block = Block() block.ParseFromString(gossip_message.content) if block.header_signature in self._seen_cache: LOGGER.debug("Drop already validated block: %s cache=%s", block.header_signature[:8], len(self._seen_cache) ) #[sig[:8] for sig in self._seen_cache]) return HandlerResult(status=HandlerStatus.DROP) if not is_valid_block(block): LOGGER.debug("block signature is invalid: %s", block.header_signature[:8]) return HandlerResult(status=HandlerStatus.DROP) self._seen_cache[block.header_signature] = None return HandlerResult(status=HandlerStatus.PASS) elif gossip_message.content_type == GossipMessage.BATCH: batch = Batch() batch.ParseFromString(gossip_message.content) if batch.header_signature in self._seen_cache: LOGGER.debug("Drop already validated batch: %s", batch.header_signature[:8]) return HandlerResult(status=HandlerStatus.DROP) if not is_valid_batch(batch): LOGGER.debug("batch signature is invalid: %s", batch.header_signature[:8]) return HandlerResult(status=HandlerStatus.DROP) self._seen_cache[batch.header_signature] = None return HandlerResult(status=HandlerStatus.PASS) elif gossip_message.content_type == GossipMessage.BATCHES: # check batches signature LOGGER.debug("Check BATCHES signature!!!") return HandlerResult(status=HandlerStatus.PASS) elif gossip_message.content_type == GossipMessage.ENDPOINTS: # check endpoints LOGGER.debug("Check ENDPOINTS signature!!!") return HandlerResult(status=HandlerStatus.PASS) # should drop the message if it does not have a valid content_type return HandlerResult(status=HandlerStatus.DROP)
def handle(self, connection_id, message_content): block_response_message = GossipBlockResponse() block_response_message.ParseFromString(message_content) block = Block() block.ParseFromString(block_response_message.content) status = validate_block(block) if status is True: LOGGER.debug("requested block passes signature verification %s", block.header_signature) return HandlerResult(status=HandlerStatus.PASS) else: LOGGER.debug("requested block's signature is invalid: %s", block.header_signature) return HandlerResult(status=HandlerStatus.DROP)
def _create_block(self): return BlockWrapper.wrap( Block(header_signature='some_block_id', batches=[], header=BlockHeader(block_num=0, previous_block_id=NULL_BLOCK_IDENTIFIER). SerializeToString()))
def gen_block(self, block_id, prev_id, num, batches): return BlockWrapper( Block(header_signature=block_id, batches=batches, header=BlockHeader( block_num=num, previous_block_id=prev_id).SerializeToString()))
def add_block(self, base_id, root=b'merkle_root'.hex()): block_id = 'b' * (128 - len(base_id)) + base_id head = self.chain_head if head: previous_id = head.header_signature num = head.header.block_num + 1 else: previous_id = 'zzzzz' num = 0 signer_public_key = b'public_key' + bytes(base_id, 'utf-8') header = BlockHeader( block_num=num, previous_block_id=previous_id, signer_public_key=signer_public_key.hex(), batch_ids=[block_id], consensus=b'consensus', state_root_hash=root) block = Block( header=header.SerializeToString(), header_signature=block_id, batches=[make_mock_batch(base_id)]) self.put_blocks([block])
def gossip_message_preprocessor(message_content_bytes): gossip_message = GossipMessage() gossip_message.ParseFromString(message_content_bytes) tag = gossip_message.content_type if tag == GossipMessage.BLOCK: obj = Block() obj.ParseFromString(gossip_message.content) elif tag == GossipMessage.BATCH: obj = Batch() obj.ParseFromString(gossip_message.content) content = obj, tag, gossip_message.time_to_live return PreprocessorResult(content=content)
def chain_head_fn(self): (vec_ptr, vec_len, vec_cap) = ffi.prepare_vec_result() _libexec('chain_controller_chain_head', self.pointer, ctypes.byref(vec_ptr), ctypes.byref(vec_len), ctypes.byref(vec_cap)) # Check if NULL if not vec_ptr: return None payload = ffi.from_rust_vec(vec_ptr, vec_len, vec_cap) block = Block() block.ParseFromString(payload) return block
def _create_blocks(self, block_count, batch_count, valid_block=True, valid_batch=True): block_list = [] for i in range(block_count): batch_list = self._create_batches(batch_count, 2, valid_batch=valid_batch) batch_ids = [batch.header_signature for batch in batch_list] block_header = BlockHeader(signer_pubkey=self.public_key, batch_ids=batch_ids) header_bytes = block_header.SerializeToString() if valid_block: signature = signing.sign(header_bytes, self.private_key) else: signature = "bad_signature" block = Block(header=header_bytes, batches=batch_list, header_signature=signature) block_list.append(block) return block_list