def __write_block_data(self, block: Block, confirm_info): # a condition for the exception case of genesis block. next_total_tx = self.__total_tx if block.header.height > 0: next_total_tx += len(block.body.transactions) bit_length = next_total_tx.bit_length() byte_length = (bit_length + 7) // 8 next_total_tx_bytes = next_total_tx.to_bytes(byte_length, byteorder='big') block_serializer = BlockSerializer.new(block.header.version, self.tx_versioner) block_serialized = json.dumps(block_serializer.serialize(block)) block_hash_encoded = block.header.hash.hex().encode(encoding='UTF-8') batch = leveldb.WriteBatch() batch.Put(block_hash_encoded, block_serialized.encode("utf-8")) batch.Put(BlockChain.LAST_BLOCK_KEY, block_hash_encoded) batch.Put(BlockChain.TRANSACTION_COUNT_KEY, next_total_tx_bytes) batch.Put( BlockChain.BLOCK_HEIGHT_KEY + block.header.height.to_bytes( conf.BLOCK_HEIGHT_BYTES_LEN, byteorder='big'), block_hash_encoded) if confirm_info: batch.Put(BlockChain.CONFIRM_INFO_KEY + block_hash_encoded, b'0x1') self.__confirmed_block_db.Write(batch) return next_total_tx
def __block_request_by_citizen(self, block_height, rs_rest_stub): try: get_block_result = rs_rest_stub.call("GetBlockByHeight", { 'channel': self.__channel_name, 'height': str(block_height) }) max_height_result = rs_rest_stub.call("Status") if max_height_result.status_code != 200: raise ConnectionError block_version = self.get_blockchain().block_versioner.get_version( block_height) block_serializer = BlockSerializer.new( block_version, self.get_blockchain().tx_versioner) block = block_serializer.deserialize(get_block_result['block']) return block, json.loads( max_height_result.text )['block_height'], message_code.Response.success except ReceivedErrorResponse as e: rs_rest_stub.update_methods_version() return self.__block_request_by_citizen(block_height, rs_rest_stub)
def put_precommit_block(self, precommit_block: Block): # write precommit block to DB logging.debug( f"blockchain:put_precommit_block ({self.__channel_name}), hash ({precommit_block.header.hash.hex()})" ) if self.__last_block.header.height < precommit_block.header.height: self.__precommit_tx(precommit_block) util.logger.spam( f"blockchain:put_precommit_block:confirmed_transaction_list") block_serializer = BlockSerializer.new( precommit_block.header.version, self.tx_versioner) block_serialized = block_serializer.serialize(precommit_block) block_serialized = json.dumps(block_serialized) block_serialized = block_serialized.encode('utf-8') results = self.__confirmed_block_db.Put( BlockChain.PRECOMMIT_BLOCK_KEY, block_serialized) util.logger.spam(f"result of to write to db ({results})") logging.info( f"ADD BLOCK PRECOMMIT HEIGHT : {precommit_block.header.height} , " f"HASH : {precommit_block.header.hash.hex()}, CHANNEL : {self.__channel_name}" ) else: results = None logging.debug( f"blockchain:put_precommit_block::this precommit block is not validate. " f"the height of precommit block must be bigger than the last block." f"(last block:{self.__last_block.header.height}/" f"precommit block:{precommit_block.header.height})") return results
async def announce_new_block(self, subscriber_block_height: int, subscriber_id: str): blockchain = self._channel_service.block_manager.get_blockchain() while True: my_block_height = blockchain.block_height if subscriber_block_height >= my_block_height: async with self._citizen_condition_new_block: await self._citizen_condition_new_block.wait() new_block_height = subscriber_block_height + 1 new_block = blockchain.find_block_by_height(new_block_height) confirm_info: bytes = blockchain.find_confirm_info_by_height( new_block_height) if new_block is None: logging.warning( f"Cannot find block height({new_block_height})") await asyncio.sleep( 0.5 ) # To prevent excessive occupancy of the CPU in an infinite loop continue logging.debug( f"announce_new_block: height({new_block.header.height}), to: {subscriber_id}" ) bs = BlockSerializer.new(new_block.header.version, blockchain.tx_versioner) return json.dumps(bs.serialize(new_block)), confirm_info
def init_blockchain(self): # level DB에서 블럭을 읽어 들이며, 만약 levelDB에 블럭이 없을 경우 제네시스 블럭을 만든다 try: last_block_key = self.__confirmed_block_db.Get( BlockChain.LAST_BLOCK_KEY, True) except KeyError: last_block_key = None logging.debug("LAST BLOCK KEY : %s", last_block_key) if last_block_key: block_dump = self.__confirmed_block_db.Get(last_block_key) block_dump = json.loads(block_dump) block_height = self.__block_versioner.get_height(block_dump) block_version = self.__block_versioner.get_version(block_height) confirm_info = self.find_confirm_info_by_hash( self.__block_versioner.get_hash(block_dump)) block_dump["confirm_prev_block"] = confirm_info is not b'' self.__last_block = BlockSerializer.new( block_version, self.tx_versioner).deserialize(block_dump) logging.debug("restore from last block hash(" + str(self.__last_block.header.hash.hex()) + ")") logging.debug("restore from last block height(" + str(self.__last_block.header.height) + ")") # 블럭의 높이는 마지막 블럭의 높이와 같음 if self.__last_block is None: self.__block_height = -1 else: self.__block_height = self.__last_block.header.height logging.debug(f"ENGINE-303 init_blockchain: {self.__block_height}")
async def announce_confirmed_block(self, serialized_block, commit_state="{}"): try: bs = BlockSerializer.new("0.1a") json_block = json.loads(serialized_block) confirmed_block = bs.deserialize(json_block) util.logger.spam(f"channel_inner_service:announce_confirmed_block\n " f"hash({confirmed_block.header.hash.hex()}) " f"block height({confirmed_block.header.height}), " f"commit_state({commit_state})") header: blocks.v0_1a.BlockHeader = confirmed_block.header if not header.commit_state: bb = BlockBuilder.from_new(confirmed_block) confirmed_block = bb.build() # to generate commit_state header = confirmed_block.header try: commit_state = ast.literal_eval(commit_state) except Exception as e: logging.warning(f"channel_inner_service:announce_confirmed_block FAIL get commit_state_dict, " f"error by : {e}") if header.commit_state != commit_state: raise RuntimeError(f"Commit states does not match. " f"Generated {header.commit_state}, Received {commit_state}") if self._channel_service.block_manager.get_blockchain().block_height < confirmed_block.header.height: self._channel_service.block_manager.add_confirmed_block(confirmed_block) else: logging.debug(f"channel_inner_service:announce_confirmed_block " f"already synced block height({confirmed_block.header.height})") response_code = message_code.Response.success except Exception as e: logging.error(f"announce confirmed block error : {e}") response_code = message_code.Response.fail return response_code
def init_block_chain(self, is_leader=False): # level DB에서 블럭을 읽어 들이며, 만약 levelDB에 블럭이 없을 경우 제네시스 블럭을 만든다 try: last_block_key = self.__confirmed_block_db.Get( BlockChain.LAST_BLOCK_KEY, True) except KeyError: last_block_key = None logging.debug("LAST BLOCK KEY : %s", last_block_key) if last_block_key: block_dump = self.__confirmed_block_db.Get(last_block_key) self.__last_block = BlockSerializer.new("0.1a").deserialize( json.loads(block_dump)) logging.debug("restore from last block hash(" + str(self.__last_block.header.hash.hex()) + ")") logging.debug("restore from last block height(" + str(self.__last_block.header.height) + ")") # 블럭의 높이는 마지막 블럭의 높이와 같음 if self.__last_block is None: self.__block_height = -1 else: self.__block_height = self.__last_block.header.height logging.debug(f"ENGINE-303 init_block_chain: {self.__block_height}")
async def announce_new_block(self, subscriber_block_height: int): blockchain = self._channel_service.block_manager.get_blockchain() while True: my_block_height = blockchain.block_height if subscriber_block_height > my_block_height: message = { 'error': "Announced block height is lower than subscriber's." } return json.dumps(message) if subscriber_block_height == my_block_height: async with self._citizen_condition_new_block: await self._citizen_condition_new_block.wait() new_block_height = subscriber_block_height + 1 new_block = blockchain.find_block_by_height(new_block_height) if new_block is None: logging.warning( f"Cannot find block height({new_block_height})") await asyncio.sleep( 0.5 ) # To prevent excessive occupancy of the CPU in an infinite loop continue logging.debug( f"announce_new_block: height({new_block.header.height}), hash({new_block.header.hash}), " f"target: {self._citizen_set}") bs = BlockSerializer.new(new_block.header.version, blockchain.tx_versioner) return json.dumps(bs.serialize(new_block))
def block_loads(self, block_dumped: bytes) -> Block: block_dumped = zlib.decompress(block_dumped) block_json = block_dumped.decode(encoding=conf.PEER_DATA_ENCODING) block_serialized = json.loads(block_json) block_height = self.__block_versioner.get_height(block_serialized) block_version = self.__block_versioner.get_version(block_height) block_serializer = BlockSerializer.new(block_version, self.tx_versioner) return block_serializer.deserialize(block_serialized)
async def get_block_v2(self, block_height, block_hash, block_data_filter, tx_data_filter): # This is a temporary function for v2 support of exchanges. block, block_filter, block_hash, fail_response_code, tx_filter = await self.__get_block( block_data_filter, block_hash, block_height, tx_data_filter) if fail_response_code: return fail_response_code, block_hash, json.dumps({}), "" bs = BlockSerializer.new(block.header.version) block_data_dict = bs.serialize(block) if block.header.height == 0: return message_code.Response.success, block_hash, json.dumps( block_data_dict), [] confirmed_tx_list = block_data_dict["confirmed_transaction_list"] confirmed_tx_list_without_fail = [] tv = TransactionVersions() tss = { "genesis": TransactionSerializer.new( "genesis", tv.get_hash_generator_version("genesis")), "0x2": TransactionSerializer.new("0x2", tv.get_hash_generator_version("0x2")), "0x3": TransactionSerializer.new("0x3", tv.get_hash_generator_version("0x3")) } for tx in confirmed_tx_list: version = tv.get_version(tx) tx_hash = tss[version].get_hash(tx) invoke_result = self._channel_service.block_manager.get_invoke_result( tx_hash) if 'failure' in invoke_result: continue if tv.get_version(tx) == "0x3": step_used, step_price = int(invoke_result["stepUsed"], 16), int( invoke_result["stepPrice"], 16) tx["fee"] = hex(step_used * step_price) confirmed_tx_list_without_fail.append(tx) # Replace the existing confirmed_tx_list with v2 ver. block_data_dict[ "confirmed_transaction_list"] = confirmed_tx_list_without_fail block_data_json = json.dumps(block_data_dict) if fail_response_code: return fail_response_code, block_hash, json.dumps({}), [] return message_code.Response.success, block_hash, block_data_json, []
def __find_block_by_key(self, key): try: block_bytes = self.__confirmed_block_db.Get(key) block_dumped = json.loads(block_bytes) return BlockSerializer.new("0.1a").deserialize(block_dumped) except KeyError as e: logging.error( f"__find_block_by_key::KeyError block_hash({key}) error({e})") return None
async def __add_confirmed_block(self, block_json: str): block_dict = json.loads(block_json) block_serializer = BlockSerializer.new(block_dict["version"]) confirmed_block = block_serializer.deserialize(block_dict) logging.debug( f"add_confirmed_block height({confirmed_block.header.height}), " f"hash({confirmed_block.header.hash.hex()})") ObjectManager().channel_service.block_manager.add_confirmed_block( confirmed_block)
async def get_block(self, block_height, block_hash, block_data_filter, tx_data_filter): block, block_filter, block_hash, fail_response_code, tx_filter = await self.__get_block( block_data_filter, block_hash, block_height, tx_data_filter) if fail_response_code: return fail_response_code, block_hash, json.dumps({}), "" bs = BlockSerializer.new(block.header.version) block_dict = bs.serialize(block) return message_code.Response.success, block_hash, json.dumps(block_dict), []
def __find_block_by_key(self, key): try: block_bytes = self.__confirmed_block_db.Get(key) block_dumped = json.loads(block_bytes) block_height = self.__block_versioner.get_height(block_dumped) block_version = self.__block_versioner.get_version(block_height) return BlockSerializer.new(block_version, self.tx_versioner).deserialize(block_dumped) except KeyError as e: logging.error(f"__find_block_by_key::KeyError block_hash({key}) error({e})") return None
async def get_block(self, block_height, block_hash, block_data_filter, tx_data_filter): block, block_filter, block_hash, confirm_info, fail_response_code, tx_filter = \ await self.__get_block(block_data_filter, block_hash, block_height, tx_data_filter) if fail_response_code: return fail_response_code, block_hash, b"", json.dumps({}), "" tx_versioner = self._channel_service.block_manager.get_blockchain( ).tx_versioner bs = BlockSerializer.new(block.header.version, tx_versioner) block_dict = bs.serialize(block) return message_code.Response.success, block_hash, confirm_info, json.dumps( block_dict), []
async def node_ws_PublishNewBlock(self, **kwargs): if 'error' in kwargs: return ObjectManager().channel_service.shutdown_peer(message=kwargs.get('error')) block_dict = kwargs.get('block') new_block_height = block_dict.get('height') if new_block_height > ObjectManager().channel_service.block_manager.get_blockchain().block_height: block_serializer = BlockSerializer.new(block_dict["version"]) confirmed_block = block_serializer.deserialize(block_dict) logging.debug(f"add_confirmed_block height({confirmed_block.header.height}), " f"hash({confirmed_block.header.hash.hex()})") ObjectManager().channel_service.block_manager.add_confirmed_block(confirmed_block)
def _rebuild_transaction_count_from_blocks(self): total_tx = 0 block_hash = self.__last_block.header.hash.hex() block_serializer = BlockSerializer.new("0.1a") while block_hash != "": block_dump = self.__confirmed_block_db.Get( block_hash.encode(encoding='UTF-8')) block = block_serializer.deserialize(json.loads(block_dump)) # Count only normal block`s tx count, not genesis block`s if block.header.height > 0: total_tx += len(block.body.transactions) block_hash = block.header.prev_hash.hex() return total_tx
def block_dumps(self, block: Block) -> bytes: block_version = self.__block_versioner.get_version(block.header.height) block_serializer = BlockSerializer.new(block_version, self.tx_versioner) block_serialized = block_serializer.serialize(block) """ FIXME: this is a workaround. confirm_prev_block is used temporarily. We will remove the attribute. If confirm_prev_block is serialized in serialize() function, it will be put in DB but we don't want it. """ if hasattr(block.body, 'confirm_prev_block'): block_serialized['confirm_prev_block'] = block.body.confirm_prev_block block_json = json.dumps(block_serialized) block_dumped = block_json.encode(encoding=conf.PEER_DATA_ENCODING) block_dumped = zlib.compress(block_dumped) return block_dumped
async def node_ws_PublishNewBlock(self, **kwargs): if 'error' in kwargs: if kwargs.get('code') in CONNECTION_FAIL_CONDITIONS: self._exception = ConnectionError(kwargs['error']) return else: return ObjectManager().channel_service.shutdown_peer( message=kwargs.get('error')) block_dict, confirm_info_str = kwargs.get('block'), kwargs.get( 'confirm_info') confirm_info = confirm_info_str.encode( "utf-8") if confirm_info_str else None blockchain = ObjectManager( ).channel_service.block_manager.get_blockchain() new_block_height = blockchain.block_versioner.get_height(block_dict) if new_block_height > blockchain.block_height: block_version = blockchain.block_versioner.get_version( new_block_height) block_serializer = BlockSerializer.new(block_version, blockchain.tx_versioner) confirmed_block = block_serializer.deserialize(block_dict) block_verifier = BlockVerifier.new(block_version, blockchain.tx_versioner) block_verifier.invoke_func = ObjectManager( ).channel_service.score_invoke reps = ObjectManager().channel_service.get_rep_ids() try: block_verifier.verify(confirmed_block, blockchain.last_block, blockchain, blockchain.last_block.header.next_leader, reps=reps) except Exception as e: self._exception = AnnounceNewBlockError( f"error: {type(e)}, message: {str(e)}") else: logging.debug( f"add_confirmed_block height({confirmed_block.header.height}), " f"hash({confirmed_block.header.hash.hex()}), confirm_info({confirm_info})" ) ObjectManager( ).channel_service.block_manager.add_confirmed_block( confirmed_block=confirmed_block, confirm_info=confirm_info)
def __block_request_by_citizen(self, block_height, rs_rest_stub): get_block_result = rs_rest_stub.call("GetBlockByHeight", { 'channel': self.__channel_name, 'height': str(block_height) }) last_block = rs_rest_stub.call("GetLastBlock") max_height = self.__blockchain.block_versioner.get_height(last_block) block_version = self.__blockchain.block_versioner.get_version( block_height) block_serializer = BlockSerializer.new( block_version, self.get_blockchain().tx_versioner) block = block_serializer.deserialize(get_block_result['block']) confirm_info = get_block_result.get('confirm_info', '') if isinstance(confirm_info, str): confirm_info = confirm_info.encode('utf-8') return block, max_height, -1, confirm_info, message_code.Response.success
def __block_request_by_citizen(self, block_height, rs_rest_stub): get_block_result = rs_rest_stub.call("GetBlockByHeight", { 'channel': self.__channel_name, 'height': str(block_height) }) max_height_result = rs_rest_stub.call("Status") if max_height_result.status_code != 200: raise ConnectionError block_version = self.get_blockchain().block_versioner.get_version( block_height) block_serializer = BlockSerializer.new( block_version, self.get_blockchain().tx_versioner) block = block_serializer.deserialize(get_block_result['block']) confirm_info = get_block_result[ 'confirm_info'] if 'confirm_info' in get_block_result else None return block, json.loads( max_height_result.text )['block_height'], confirm_info, message_code.Response.success
def __add_block(self, block: Block): with self.__add_block_lock: need_to_commit = True invoke_results = self.__invoke_results.get(block.header.hash.hex(), None) if invoke_results is None: if block.header.height == 0: block, invoke_results = ObjectManager( ).channel_service.genesis_invoke(block) else: block, invoke_results = ObjectManager( ).channel_service.score_invoke(block) try: if need_to_commit: self.__add_tx_to_block_db(block, invoke_results) ObjectManager( ).channel_service.score_write_precommit_state(block) except Exception as e: logging.warning(f"blockchain:add_block FAIL " f"channel_service.score_write_precommit_state") raise e finally: self.__invoke_results.pop(block.header.hash, None) # a condition for the exception case of genesis block. next_total_tx = self.__total_tx if block.header.height > 0: next_total_tx += len(block.body.transactions) bit_length = next_total_tx.bit_length() byte_length = (bit_length + 7) // 8 next_total_tx_bytes = next_total_tx.to_bytes(byte_length, byteorder='big') block_serializer = BlockSerializer.new("0.1a") block_serialized = json.dumps(block_serializer.serialize(block)) block_hash_encoded = block.header.hash.hex().encode( encoding='UTF-8') batch = leveldb.WriteBatch() batch.Put(block_hash_encoded, block_serialized.encode("utf-8")) batch.Put(BlockChain.LAST_BLOCK_KEY, block_hash_encoded) batch.Put(BlockChain.TRANSACTION_COUNT_KEY, next_total_tx_bytes) batch.Put( BlockChain.BLOCK_HEIGHT_KEY + block.header.height.to_bytes( conf.BLOCK_HEIGHT_BYTES_LEN, byteorder='big'), block_hash_encoded) self.__confirmed_block_db.Write(batch) self.__last_block = block self.__block_height = self.__last_block.header.height self.__total_tx = next_total_tx logging.debug( f"blockchain add_block set block_height({self.__block_height}), " f"last_block({self.__last_block.header.hash.hex()})") logging.info(f"ADD BLOCK HEIGHT : {block.header.height} , " f"HASH : {block.header.hash.hex()} , " f"CHANNEL : {self.__channel_name}") util.apm_event( self.__peer_id, { 'event_type': 'AddBlock', 'peer_id': self.__peer_id, 'peer_name': conf.PEER_NAME, 'channel_name': self.__channel_name, 'data': { 'block_height': self.__block_height } }) # notify new block ObjectManager().channel_service.inner_service.notify_new_block() return True
def test_block_v0_3(self): private_auth = test_util.create_default_peer_auth() tx_versioner = TransactionVersioner() dummy_receipts = {} block_builder = BlockBuilder.new("0.3", tx_versioner) for i in range(1000): tx_builder = TransactionBuilder.new("0x3", tx_versioner) tx_builder.private_key = private_auth.private_key tx_builder.to_address = ExternalAddress.new() tx_builder.step_limit = random.randint(0, 10000) tx_builder.value = random.randint(0, 10000) tx_builder.nid = 2 tx = tx_builder.build() tx_serializer = TransactionSerializer.new(tx.version, tx_versioner) block_builder.transactions[tx.hash] = tx dummy_receipts[tx.hash.hex()] = { "dummy_receipt": "dummy", "tx_dumped": tx_serializer.to_full_data(tx) } block_builder.peer_private_key = private_auth.private_key block_builder.height = 0 block_builder.state_hash = Hash32(bytes(Hash32.size)) block_builder.receipts = dummy_receipts block_builder.reps = [ ExternalAddress.fromhex_address(private_auth.address) ] block_builder.next_leader = ExternalAddress.fromhex( "hx00112233445566778899aabbccddeeff00112233") block = block_builder.build() block_verifier = BlockVerifier.new("0.3", tx_versioner) block_verifier.invoke_func = lambda b: (block, dummy_receipts) block_verifier.verify(block, None, None, block.header.peer_id, reps=block_builder.reps) block_serializer = BlockSerializer.new("0.3", tx_versioner) block_serialized = block_serializer.serialize(block) block_deserialized = block_serializer.deserialize(block_serialized) assert block.header == block_deserialized.header # FIXME : confirm_prev_block not serialized # assert block.body == block_deserialized.body tx_hashes = list(block.body.transactions) tx_index = random.randrange(0, len(tx_hashes)) block_prover = BlockProver.new(block.header.version, tx_hashes, BlockProverType.Transaction) tx_proof = block_prover.get_proof(tx_index) assert block_prover.prove(tx_hashes[tx_index], block.header.transaction_hash, tx_proof) block_prover = BlockProver.new(block.header.version, block_builder.receipts, BlockProverType.Receipt) receipt_proof = block_prover.get_proof(tx_index) receipt_hash = block_prover.to_hash32(block_builder.receipts[tx_index]) assert block_prover.prove(receipt_hash, block.header.receipt_hash, receipt_proof)