def __write_block_data(self, block: Block, confirm_info):
# a condition for the exception case of genesis block.
next_total_tx = self.__total_tx
if block.header.height > 0:
next_total_tx += len(block.body.transactions)
bit_length = next_total_tx.bit_length()
byte_length = (bit_length + 7) // 8
next_total_tx_bytes = next_total_tx.to_bytes(byte_length,
byteorder='big')
block_serializer = BlockSerializer.new(block.header.version,
self.tx_versioner)
block_serialized = json.dumps(block_serializer.serialize(block))
block_hash_encoded = block.header.hash.hex().encode(encoding='UTF-8')
batch = leveldb.WriteBatch()
batch.Put(block_hash_encoded, block_serialized.encode("utf-8"))
batch.Put(BlockChain.LAST_BLOCK_KEY, block_hash_encoded)
batch.Put(BlockChain.TRANSACTION_COUNT_KEY, next_total_tx_bytes)
batch.Put(
BlockChain.BLOCK_HEIGHT_KEY + block.header.height.to_bytes(
conf.BLOCK_HEIGHT_BYTES_LEN, byteorder='big'),
block_hash_encoded)
if confirm_info:
batch.Put(BlockChain.CONFIRM_INFO_KEY + block_hash_encoded, b'0x1')
self.__confirmed_block_db.Write(batch)
return next_total_tx
def __block_request_by_citizen(self, block_height, rs_rest_stub):
try:
get_block_result = rs_rest_stub.call("GetBlockByHeight", {
'channel': self.__channel_name,
'height': str(block_height)
})
max_height_result = rs_rest_stub.call("Status")
if max_height_result.status_code != 200:
raise ConnectionError
block_version = self.get_blockchain().block_versioner.get_version(
block_height)
block_serializer = BlockSerializer.new(
block_version,
self.get_blockchain().tx_versioner)
block = block_serializer.deserialize(get_block_result['block'])
return block, json.loads(
max_height_result.text
)['block_height'], message_code.Response.success
except ReceivedErrorResponse as e:
rs_rest_stub.update_methods_version()
return self.__block_request_by_citizen(block_height, rs_rest_stub)
def put_precommit_block(self, precommit_block: Block):
# write precommit block to DB
logging.debug(
f"blockchain:put_precommit_block ({self.__channel_name}), hash ({precommit_block.header.hash.hex()})"
)
if self.__last_block.header.height < precommit_block.header.height:
self.__precommit_tx(precommit_block)
util.logger.spam(
f"blockchain:put_precommit_block:confirmed_transaction_list")
block_serializer = BlockSerializer.new(
precommit_block.header.version, self.tx_versioner)
block_serialized = block_serializer.serialize(precommit_block)
block_serialized = json.dumps(block_serialized)
block_serialized = block_serialized.encode('utf-8')
results = self.__confirmed_block_db.Put(
BlockChain.PRECOMMIT_BLOCK_KEY, block_serialized)
util.logger.spam(f"result of to write to db ({results})")
logging.info(
f"ADD BLOCK PRECOMMIT HEIGHT : {precommit_block.header.height} , "
f"HASH : {precommit_block.header.hash.hex()}, CHANNEL : {self.__channel_name}"
)
else:
results = None
logging.debug(
f"blockchain:put_precommit_block::this precommit block is not validate. "
f"the height of precommit block must be bigger than the last block."
f"(last block:{self.__last_block.header.height}/"
f"precommit block:{precommit_block.header.height})")
return results
async def announce_new_block(self, subscriber_block_height: int,
subscriber_id: str):
blockchain = self._channel_service.block_manager.get_blockchain()
while True:
my_block_height = blockchain.block_height
if subscriber_block_height >= my_block_height:
async with self._citizen_condition_new_block:
await self._citizen_condition_new_block.wait()
new_block_height = subscriber_block_height + 1
new_block = blockchain.find_block_by_height(new_block_height)
confirm_info: bytes = blockchain.find_confirm_info_by_height(
new_block_height)
if new_block is None:
logging.warning(
f"Cannot find block height({new_block_height})")
await asyncio.sleep(
0.5
) # To prevent excessive occupancy of the CPU in an infinite loop
continue
logging.debug(
f"announce_new_block: height({new_block.header.height}), to: {subscriber_id}"
)
bs = BlockSerializer.new(new_block.header.version,
blockchain.tx_versioner)
return json.dumps(bs.serialize(new_block)), confirm_info
def init_blockchain(self):
# level DB에서 블럭을 읽어 들이며, 만약 levelDB에 블럭이 없을 경우 제네시스 블럭을 만든다
try:
last_block_key = self.__confirmed_block_db.Get(
BlockChain.LAST_BLOCK_KEY, True)
except KeyError:
last_block_key = None
logging.debug("LAST BLOCK KEY : %s", last_block_key)
if last_block_key:
block_dump = self.__confirmed_block_db.Get(last_block_key)
block_dump = json.loads(block_dump)
block_height = self.__block_versioner.get_height(block_dump)
block_version = self.__block_versioner.get_version(block_height)
confirm_info = self.find_confirm_info_by_hash(
self.__block_versioner.get_hash(block_dump))
block_dump["confirm_prev_block"] = confirm_info is not b''
self.__last_block = BlockSerializer.new(
block_version, self.tx_versioner).deserialize(block_dump)
logging.debug("restore from last block hash(" +
str(self.__last_block.header.hash.hex()) + ")")
logging.debug("restore from last block height(" +
str(self.__last_block.header.height) + ")")
# 블럭의 높이는 마지막 블럭의 높이와 같음
if self.__last_block is None:
self.__block_height = -1
else:
self.__block_height = self.__last_block.header.height
logging.debug(f"ENGINE-303 init_blockchain: {self.__block_height}")
async def announce_confirmed_block(self, serialized_block, commit_state="{}"):
try:
bs = BlockSerializer.new("0.1a")
json_block = json.loads(serialized_block)
confirmed_block = bs.deserialize(json_block)
util.logger.spam(f"channel_inner_service:announce_confirmed_block\n "
f"hash({confirmed_block.header.hash.hex()}) "
f"block height({confirmed_block.header.height}), "
f"commit_state({commit_state})")
header: blocks.v0_1a.BlockHeader = confirmed_block.header
if not header.commit_state:
bb = BlockBuilder.from_new(confirmed_block)
confirmed_block = bb.build() # to generate commit_state
header = confirmed_block.header
try:
commit_state = ast.literal_eval(commit_state)
except Exception as e:
logging.warning(f"channel_inner_service:announce_confirmed_block FAIL get commit_state_dict, "
f"error by : {e}")
if header.commit_state != commit_state:
raise RuntimeError(f"Commit states does not match. "
f"Generated {header.commit_state}, Received {commit_state}")
if self._channel_service.block_manager.get_blockchain().block_height < confirmed_block.header.height:
self._channel_service.block_manager.add_confirmed_block(confirmed_block)
else:
logging.debug(f"channel_inner_service:announce_confirmed_block "
f"already synced block height({confirmed_block.header.height})")
response_code = message_code.Response.success
except Exception as e:
logging.error(f"announce confirmed block error : {e}")
response_code = message_code.Response.fail
return response_code
def init_block_chain(self, is_leader=False):
# level DB에서 블럭을 읽어 들이며, 만약 levelDB에 블럭이 없을 경우 제네시스 블럭을 만든다
try:
last_block_key = self.__confirmed_block_db.Get(
BlockChain.LAST_BLOCK_KEY, True)
except KeyError:
last_block_key = None
logging.debug("LAST BLOCK KEY : %s", last_block_key)
if last_block_key:
block_dump = self.__confirmed_block_db.Get(last_block_key)
self.__last_block = BlockSerializer.new("0.1a").deserialize(
json.loads(block_dump))
logging.debug("restore from last block hash(" +
str(self.__last_block.header.hash.hex()) + ")")
logging.debug("restore from last block height(" +
str(self.__last_block.header.height) + ")")
# 블럭의 높이는 마지막 블럭의 높이와 같음
if self.__last_block is None:
self.__block_height = -1
else:
self.__block_height = self.__last_block.header.height
logging.debug(f"ENGINE-303 init_block_chain: {self.__block_height}")
async def announce_new_block(self, subscriber_block_height: int):
blockchain = self._channel_service.block_manager.get_blockchain()
while True:
my_block_height = blockchain.block_height
if subscriber_block_height > my_block_height:
message = {
'error':
"Announced block height is lower than subscriber's."
}
return json.dumps(message)
if subscriber_block_height == my_block_height:
async with self._citizen_condition_new_block:
await self._citizen_condition_new_block.wait()
new_block_height = subscriber_block_height + 1
new_block = blockchain.find_block_by_height(new_block_height)
if new_block is None:
logging.warning(
f"Cannot find block height({new_block_height})")
await asyncio.sleep(
0.5
) # To prevent excessive occupancy of the CPU in an infinite loop
continue
logging.debug(
f"announce_new_block: height({new_block.header.height}), hash({new_block.header.hash}), "
f"target: {self._citizen_set}")
bs = BlockSerializer.new(new_block.header.version,
blockchain.tx_versioner)
return json.dumps(bs.serialize(new_block))
def block_loads(self, block_dumped: bytes) -> Block:
block_dumped = zlib.decompress(block_dumped)
block_json = block_dumped.decode(encoding=conf.PEER_DATA_ENCODING)
block_serialized = json.loads(block_json)
block_height = self.__block_versioner.get_height(block_serialized)
block_version = self.__block_versioner.get_version(block_height)
block_serializer = BlockSerializer.new(block_version, self.tx_versioner)
return block_serializer.deserialize(block_serialized)
async def get_block_v2(self, block_height, block_hash, block_data_filter,
tx_data_filter):
# This is a temporary function for v2 support of exchanges.
block, block_filter, block_hash, fail_response_code, tx_filter = await self.__get_block(
block_data_filter, block_hash, block_height, tx_data_filter)
if fail_response_code:
return fail_response_code, block_hash, json.dumps({}), ""
bs = BlockSerializer.new(block.header.version)
block_data_dict = bs.serialize(block)
if block.header.height == 0:
return message_code.Response.success, block_hash, json.dumps(
block_data_dict), []
confirmed_tx_list = block_data_dict["confirmed_transaction_list"]
confirmed_tx_list_without_fail = []
tv = TransactionVersions()
tss = {
"genesis":
TransactionSerializer.new(
"genesis", tv.get_hash_generator_version("genesis")),
"0x2":
TransactionSerializer.new("0x2",
tv.get_hash_generator_version("0x2")),
"0x3":
TransactionSerializer.new("0x3",
tv.get_hash_generator_version("0x3"))
}
for tx in confirmed_tx_list:
version = tv.get_version(tx)
tx_hash = tss[version].get_hash(tx)
invoke_result = self._channel_service.block_manager.get_invoke_result(
tx_hash)
if 'failure' in invoke_result:
continue
if tv.get_version(tx) == "0x3":
step_used, step_price = int(invoke_result["stepUsed"],
16), int(
invoke_result["stepPrice"], 16)
tx["fee"] = hex(step_used * step_price)
confirmed_tx_list_without_fail.append(tx)
# Replace the existing confirmed_tx_list with v2 ver.
block_data_dict[
"confirmed_transaction_list"] = confirmed_tx_list_without_fail
block_data_json = json.dumps(block_data_dict)
if fail_response_code:
return fail_response_code, block_hash, json.dumps({}), []
return message_code.Response.success, block_hash, block_data_json, []
def __find_block_by_key(self, key):
try:
block_bytes = self.__confirmed_block_db.Get(key)
block_dumped = json.loads(block_bytes)
return BlockSerializer.new("0.1a").deserialize(block_dumped)
except KeyError as e:
logging.error(
f"__find_block_by_key::KeyError block_hash({key}) error({e})")
return None
async def __add_confirmed_block(self, block_json: str):
block_dict = json.loads(block_json)
block_serializer = BlockSerializer.new(block_dict["version"])
confirmed_block = block_serializer.deserialize(block_dict)
logging.debug(
f"add_confirmed_block height({confirmed_block.header.height}), "
f"hash({confirmed_block.header.hash.hex()})")
ObjectManager().channel_service.block_manager.add_confirmed_block(
confirmed_block)
async def get_block(self, block_height, block_hash, block_data_filter, tx_data_filter):
block, block_filter, block_hash, fail_response_code, tx_filter = await self.__get_block(
block_data_filter, block_hash, block_height, tx_data_filter)
if fail_response_code:
return fail_response_code, block_hash, json.dumps({}), ""
bs = BlockSerializer.new(block.header.version)
block_dict = bs.serialize(block)
return message_code.Response.success, block_hash, json.dumps(block_dict), []
def __find_block_by_key(self, key):
try:
block_bytes = self.__confirmed_block_db.Get(key)
block_dumped = json.loads(block_bytes)
block_height = self.__block_versioner.get_height(block_dumped)
block_version = self.__block_versioner.get_version(block_height)
return BlockSerializer.new(block_version, self.tx_versioner).deserialize(block_dumped)
except KeyError as e:
logging.error(f"__find_block_by_key::KeyError block_hash({key}) error({e})")
return None
async def get_block(self, block_height, block_hash, block_data_filter,
tx_data_filter):
block, block_filter, block_hash, confirm_info, fail_response_code, tx_filter = \
await self.__get_block(block_data_filter, block_hash, block_height, tx_data_filter)
if fail_response_code:
return fail_response_code, block_hash, b"", json.dumps({}), ""
tx_versioner = self._channel_service.block_manager.get_blockchain(
).tx_versioner
bs = BlockSerializer.new(block.header.version, tx_versioner)
block_dict = bs.serialize(block)
return message_code.Response.success, block_hash, confirm_info, json.dumps(
block_dict), []
async def node_ws_PublishNewBlock(self, **kwargs):
if 'error' in kwargs:
return ObjectManager().channel_service.shutdown_peer(message=kwargs.get('error'))
block_dict = kwargs.get('block')
new_block_height = block_dict.get('height')
if new_block_height > ObjectManager().channel_service.block_manager.get_blockchain().block_height:
block_serializer = BlockSerializer.new(block_dict["version"])
confirmed_block = block_serializer.deserialize(block_dict)
logging.debug(f"add_confirmed_block height({confirmed_block.header.height}), "
f"hash({confirmed_block.header.hash.hex()})")
ObjectManager().channel_service.block_manager.add_confirmed_block(confirmed_block)
def _rebuild_transaction_count_from_blocks(self):
total_tx = 0
block_hash = self.__last_block.header.hash.hex()
block_serializer = BlockSerializer.new("0.1a")
while block_hash != "":
block_dump = self.__confirmed_block_db.Get(
block_hash.encode(encoding='UTF-8'))
block = block_serializer.deserialize(json.loads(block_dump))
# Count only normal block`s tx count, not genesis block`s
if block.header.height > 0:
total_tx += len(block.body.transactions)
block_hash = block.header.prev_hash.hex()
return total_tx
def block_dumps(self, block: Block) -> bytes:
block_version = self.__block_versioner.get_version(block.header.height)
block_serializer = BlockSerializer.new(block_version, self.tx_versioner)
block_serialized = block_serializer.serialize(block)
"""
FIXME: this is a workaround. confirm_prev_block is used temporarily. We will remove the attribute.
If confirm_prev_block is serialized in serialize() function, it will be put in DB but we don't want it.
"""
if hasattr(block.body, 'confirm_prev_block'):
block_serialized['confirm_prev_block'] = block.body.confirm_prev_block
block_json = json.dumps(block_serialized)
block_dumped = block_json.encode(encoding=conf.PEER_DATA_ENCODING)
block_dumped = zlib.compress(block_dumped)
return block_dumped
async def node_ws_PublishNewBlock(self, **kwargs):
if 'error' in kwargs:
if kwargs.get('code') in CONNECTION_FAIL_CONDITIONS:
self._exception = ConnectionError(kwargs['error'])
return
else:
return ObjectManager().channel_service.shutdown_peer(
message=kwargs.get('error'))
block_dict, confirm_info_str = kwargs.get('block'), kwargs.get(
'confirm_info')
confirm_info = confirm_info_str.encode(
"utf-8") if confirm_info_str else None
blockchain = ObjectManager(
).channel_service.block_manager.get_blockchain()
new_block_height = blockchain.block_versioner.get_height(block_dict)
if new_block_height > blockchain.block_height:
block_version = blockchain.block_versioner.get_version(
new_block_height)
block_serializer = BlockSerializer.new(block_version,
blockchain.tx_versioner)
confirmed_block = block_serializer.deserialize(block_dict)
block_verifier = BlockVerifier.new(block_version,
blockchain.tx_versioner)
block_verifier.invoke_func = ObjectManager(
).channel_service.score_invoke
reps = ObjectManager().channel_service.get_rep_ids()
try:
block_verifier.verify(confirmed_block,
blockchain.last_block,
blockchain,
blockchain.last_block.header.next_leader,
reps=reps)
except Exception as e:
self._exception = AnnounceNewBlockError(
f"error: {type(e)}, message: {str(e)}")
else:
logging.debug(
f"add_confirmed_block height({confirmed_block.header.height}), "
f"hash({confirmed_block.header.hash.hex()}), confirm_info({confirm_info})"
)
ObjectManager(
).channel_service.block_manager.add_confirmed_block(
confirmed_block=confirmed_block, confirm_info=confirm_info)
def __block_request_by_citizen(self, block_height, rs_rest_stub):
get_block_result = rs_rest_stub.call("GetBlockByHeight", {
'channel': self.__channel_name,
'height': str(block_height)
})
last_block = rs_rest_stub.call("GetLastBlock")
max_height = self.__blockchain.block_versioner.get_height(last_block)
block_version = self.__blockchain.block_versioner.get_version(
block_height)
block_serializer = BlockSerializer.new(
block_version,
self.get_blockchain().tx_versioner)
block = block_serializer.deserialize(get_block_result['block'])
confirm_info = get_block_result.get('confirm_info', '')
if isinstance(confirm_info, str):
confirm_info = confirm_info.encode('utf-8')
return block, max_height, -1, confirm_info, message_code.Response.success
def __block_request_by_citizen(self, block_height, rs_rest_stub):
get_block_result = rs_rest_stub.call("GetBlockByHeight", {
'channel': self.__channel_name,
'height': str(block_height)
})
max_height_result = rs_rest_stub.call("Status")
if max_height_result.status_code != 200:
raise ConnectionError
block_version = self.get_blockchain().block_versioner.get_version(
block_height)
block_serializer = BlockSerializer.new(
block_version,
self.get_blockchain().tx_versioner)
block = block_serializer.deserialize(get_block_result['block'])
confirm_info = get_block_result[
'confirm_info'] if 'confirm_info' in get_block_result else None
return block, json.loads(
max_height_result.text
)['block_height'], confirm_info, message_code.Response.success
def __add_block(self, block: Block):
with self.__add_block_lock:
need_to_commit = True
invoke_results = self.__invoke_results.get(block.header.hash.hex(),
None)
if invoke_results is None:
if block.header.height == 0:
block, invoke_results = ObjectManager(
).channel_service.genesis_invoke(block)
else:
block, invoke_results = ObjectManager(
).channel_service.score_invoke(block)
try:
if need_to_commit:
self.__add_tx_to_block_db(block, invoke_results)
ObjectManager(
).channel_service.score_write_precommit_state(block)
except Exception as e:
logging.warning(f"blockchain:add_block FAIL "
f"channel_service.score_write_precommit_state")
raise e
finally:
self.__invoke_results.pop(block.header.hash, None)
# a condition for the exception case of genesis block.
next_total_tx = self.__total_tx
if block.header.height > 0:
next_total_tx += len(block.body.transactions)
bit_length = next_total_tx.bit_length()
byte_length = (bit_length + 7) // 8
next_total_tx_bytes = next_total_tx.to_bytes(byte_length,
byteorder='big')
block_serializer = BlockSerializer.new("0.1a")
block_serialized = json.dumps(block_serializer.serialize(block))
block_hash_encoded = block.header.hash.hex().encode(
encoding='UTF-8')
batch = leveldb.WriteBatch()
batch.Put(block_hash_encoded, block_serialized.encode("utf-8"))
batch.Put(BlockChain.LAST_BLOCK_KEY, block_hash_encoded)
batch.Put(BlockChain.TRANSACTION_COUNT_KEY, next_total_tx_bytes)
batch.Put(
BlockChain.BLOCK_HEIGHT_KEY + block.header.height.to_bytes(
conf.BLOCK_HEIGHT_BYTES_LEN, byteorder='big'),
block_hash_encoded)
self.__confirmed_block_db.Write(batch)
self.__last_block = block
self.__block_height = self.__last_block.header.height
self.__total_tx = next_total_tx
logging.debug(
f"blockchain add_block set block_height({self.__block_height}), "
f"last_block({self.__last_block.header.hash.hex()})")
logging.info(f"ADD BLOCK HEIGHT : {block.header.height} , "
f"HASH : {block.header.hash.hex()} , "
f"CHANNEL : {self.__channel_name}")
util.apm_event(
self.__peer_id, {
'event_type': 'AddBlock',
'peer_id': self.__peer_id,
'peer_name': conf.PEER_NAME,
'channel_name': self.__channel_name,
'data': {
'block_height': self.__block_height
}
})
# notify new block
ObjectManager().channel_service.inner_service.notify_new_block()
return True
def test_block_v0_3(self):
private_auth = test_util.create_default_peer_auth()
tx_versioner = TransactionVersioner()
dummy_receipts = {}
block_builder = BlockBuilder.new("0.3", tx_versioner)
for i in range(1000):
tx_builder = TransactionBuilder.new("0x3", tx_versioner)
tx_builder.private_key = private_auth.private_key
tx_builder.to_address = ExternalAddress.new()
tx_builder.step_limit = random.randint(0, 10000)
tx_builder.value = random.randint(0, 10000)
tx_builder.nid = 2
tx = tx_builder.build()
tx_serializer = TransactionSerializer.new(tx.version, tx_versioner)
block_builder.transactions[tx.hash] = tx
dummy_receipts[tx.hash.hex()] = {
"dummy_receipt": "dummy",
"tx_dumped": tx_serializer.to_full_data(tx)
}
block_builder.peer_private_key = private_auth.private_key
block_builder.height = 0
block_builder.state_hash = Hash32(bytes(Hash32.size))
block_builder.receipts = dummy_receipts
block_builder.reps = [
ExternalAddress.fromhex_address(private_auth.address)
]
block_builder.next_leader = ExternalAddress.fromhex(
"hx00112233445566778899aabbccddeeff00112233")
block = block_builder.build()
block_verifier = BlockVerifier.new("0.3", tx_versioner)
block_verifier.invoke_func = lambda b: (block, dummy_receipts)
block_verifier.verify(block,
None,
None,
block.header.peer_id,
reps=block_builder.reps)
block_serializer = BlockSerializer.new("0.3", tx_versioner)
block_serialized = block_serializer.serialize(block)
block_deserialized = block_serializer.deserialize(block_serialized)
assert block.header == block_deserialized.header
# FIXME : confirm_prev_block not serialized
# assert block.body == block_deserialized.body
tx_hashes = list(block.body.transactions)
tx_index = random.randrange(0, len(tx_hashes))
block_prover = BlockProver.new(block.header.version, tx_hashes,
BlockProverType.Transaction)
tx_proof = block_prover.get_proof(tx_index)
assert block_prover.prove(tx_hashes[tx_index],
block.header.transaction_hash, tx_proof)
block_prover = BlockProver.new(block.header.version,
block_builder.receipts,
BlockProverType.Receipt)
receipt_proof = block_prover.get_proof(tx_index)
receipt_hash = block_prover.to_hash32(block_builder.receipts[tx_index])
assert block_prover.prove(receipt_hash, block.header.receipt_hash,
receipt_proof)
