async def node_ws_PublishNewBlock(self, **kwargs):
block_dict, votes_dumped = kwargs.get('block'), kwargs.get('confirm_info', '')
try:
votes_serialized = json.loads(votes_dumped)
vote = BlockVotes.deserialize_votes(votes_serialized)
except json.JSONDecodeError:
vote = votes_dumped
blockchain = ObjectManager().channel_service.block_manager.blockchain
new_block_height = blockchain.block_versioner.get_height(block_dict)
if new_block_height > blockchain.block_height:
block_version = blockchain.block_versioner.get_version(new_block_height)
block_serializer = BlockSerializer.new(block_version, blockchain.tx_versioner)
confirmed_block = block_serializer.deserialize(block_dict)
block_verifier = BlockVerifier.new(block_version, blockchain.tx_versioner)
block_verifier.invoke_func = blockchain.score_invoke
reps_getter = blockchain.find_preps_addresses_by_roothash
try:
block_verifier.verify(confirmed_block,
blockchain.last_block,
blockchain,
generator=blockchain.get_expected_generator(confirmed_block),
reps_getter=reps_getter)
except Exception as e:
self._exception = AnnounceNewBlockError(f"error: {type(e)}, message: {str(e)}")
else:
logging.debug(f"add_confirmed_block height({confirmed_block.header.height}), "
f"hash({confirmed_block.header.hash.hex()}), votes_dumped({votes_dumped})")
ObjectManager().channel_service.block_manager.add_confirmed_block(confirmed_block=confirmed_block,
confirm_info=vote)
finally:
ObjectManager().channel_service.reset_block_monitoring_timer()
def _vote(self, unconfirmed_block: Block, round_: int):
exc = None
try:
block_version = self.blockchain.block_versioner.get_version(unconfirmed_block.header.height)
block_verifier = BlockVerifier.new(block_version, self.blockchain.tx_versioner)
block_verifier.invoke_func = self.blockchain.score_invoke
reps_getter = self.blockchain.find_preps_addresses_by_roothash
util.logger.debug(f"unconfirmed_block.header({unconfirmed_block.header})")
block_verifier.verify(unconfirmed_block,
self.blockchain.last_block,
self.blockchain,
generator=self.blockchain.get_expected_generator(unconfirmed_block),
reps_getter=reps_getter)
except NotInReps as e:
util.logger.debug(f"Not In Reps({e}) state({self.__channel_service.state_machine.state})")
except BlockHeightMismatch as e:
exc = e
util.logger.warning(f"Don't vote to the block of unexpected height. {e!r}")
except Exception as e:
exc = e
util.logger.exception(f"{e!r}")
else:
self.candidate_blocks.add_block(
unconfirmed_block, self.blockchain.find_preps_addresses_by_header(unconfirmed_block.header))
finally:
if isinstance(exc, BlockHeightMismatch):
return
is_validated = exc is None
vote = self.vote_unconfirmed_block(unconfirmed_block, round_, is_validated)
if self.__channel_service.state_machine.state == "BlockGenerate" and self.consensus_algorithm:
self.consensus_algorithm.vote(vote)
def test_valid_timestamp(self):
"""Test for timestamp buffer in block verifier"""
def block_maker(timestamp: int, height: int = 0, prev_hash=None):
"""Make dummy block"""
tx_versioner = TransactionVersioner()
dummy_receipts = {}
block_builder = BlockBuilder.new("0.1a", tx_versioner)
for i in range(1000):
tx_builder = TransactionBuilder.new("0x3", None, tx_versioner)
tx_builder.signer = test_signer
tx_builder.to_address = ExternalAddress.new()
tx_builder.step_limit = random.randint(0, 10000)
tx_builder.value = random.randint(0, 10000)
tx_builder.nid = 2
tx = tx_builder.build()
tx_serializer = TransactionSerializer.new(tx.version, tx.type(), tx_versioner)
block_builder.transactions[tx.hash] = tx
dummy_receipts[tx.hash.hex()] = {
"dummy_receipt": "dummy",
"tx_dumped": tx_serializer.to_full_data(tx)
}
block_builder.signer = test_signer
block_builder.prev_hash = prev_hash
block_builder.height = height
block_builder.state_hash = Hash32(bytes(Hash32.size))
block_builder.receipts = dummy_receipts
block_builder.reps = [ExternalAddress.fromhex_address(test_signer.address)]
block_builder.peer_id = ExternalAddress.fromhex(test_signer.address)
block_builder.next_leader = ExternalAddress.fromhex(test_signer.address)
block_builder.fixed_timestamp = timestamp
b = block_builder.build()
assert b.header.timestamp == timestamp
return b
test_signer = Signer.from_prikey(os.urandom(32))
first_block = block_maker(height=0, timestamp=utils.get_time_stamp())
second_block = block_maker(height=1, timestamp=utils.get_time_stamp() + 5, prev_hash=first_block.header.hash)
third_block_from_far_future = block_maker(height=2, prev_hash=second_block.header.hash,
timestamp=utils.get_time_stamp() + conf.TIMESTAMP_BUFFER_IN_VERIFIER + 5_000_000)
block_verifier = BlockVerifier.new("0.1a", TransactionVersioner())
leader = first_block.header.peer_id
reps = [ExternalAddress.fromhex_address(test_signer.address)]
print("*---Normal time range")
block_verifier.verify(block=second_block, prev_block=first_block,
blockchain=None, generator=leader, reps=reps)
print("*---Abnormal time range")
with self.assertRaises(Exception):
block_verifier.verify(block=third_block_from_far_future, prev_block=second_block,
blockchain=None, generator=leader, reps=reps)
def test_verify(self, plyvel_db, block_versioner, tx_versioner):
"""
1. prepare plyvel db, block_versioner, tx_versioner
2. pick block, transaction, vote, etc from db
3. verify block, vote transaction, vote, etc...
"""
# given db instance, block_versioner, tx_versioner
block_key = plyvel_db.get(b'last_block_key')
while True:
# when get block from db
block_dumped = plyvel_db.get(block_key)
Logger.info(f"block_dump : {block_dumped}")
block_serialized = json.loads(block_dumped)
block_height = block_versioner.get_height(block_serialized)
block_version = block_versioner.get_version(block_height)
block_serializer = BlockSerializer.new(block_version, tx_versioner)
block = block_serializer.deserialize(block_serialized)
Logger.info(f"block_height : {block_height}, block_version : {block_version}")
if block_height == 0:
break
# then block verify
block_verifier = BlockVerifier.new(block_version, tx_versioner)
block_verifier.verify_signature(block)
# then vote verify
if parse_version(block_version) >= parse_version("0.3"):
Logger.info(f"leader_votes : {block.body.leader_votes}")
for leader_vote in block.body.leader_votes:
if not leader_vote:
continue
leader_vote.verify()
Logger.info(f"prev_votes : {block.body.prev_votes}")
for block_vote in block.body.prev_votes:
if not block_vote:
continue
block_vote.verify()
# then transaction verify
for tx in block.body.transactions.values():
tv = TransactionVerifier.new(tx.version, tx.type(), tx_versioner)
tv.verify_signature(tx)
Logger.info(f"prev_hash : {block.header.prev_hash}, {bytes(block.header.prev_hash)}")
block_key = block.header.prev_hash.hex().encode("utf-8")
def verify_confirm_info(self, unconfirmed_block: Block):
unconfirmed_header = unconfirmed_block.header
my_height = self.blockchain.block_height
if my_height < (unconfirmed_header.height - 2):
raise ConfirmInfoInvalidNeedBlockSync(
f"trigger block sync: my_height({my_height}), "
f"unconfirmed_block.header.height({unconfirmed_header.height})"
)
is_rep = ObjectManager().channel_service.is_support_node_function(
conf.NodeFunction.Vote)
if is_rep and my_height == unconfirmed_header.height - 2 and not self.blockchain.last_unconfirmed_block:
raise ConfirmInfoInvalidNeedBlockSync(
f"trigger block sync: my_height({my_height}), "
f"unconfirmed_block.header.height({unconfirmed_header.height})"
)
# a block is already added that same height unconfirmed_block height
if my_height >= unconfirmed_header.height:
raise ConfirmInfoInvalidAddedBlock(
f"block is already added my_height({my_height}), "
f"unconfirmed_block.header.height({unconfirmed_header.height})"
)
block_verifier = BlockVerifier.new(unconfirmed_header.version,
self.blockchain.tx_versioner)
prev_block = self.blockchain.get_prev_block(unconfirmed_block)
reps_getter = self.blockchain.find_preps_addresses_by_roothash
util.logger.spam(
f"prev_block: {prev_block.header.hash if prev_block else None}")
if not prev_block:
raise NotReadyToConfirmInfo(
"There is no prev block or not ready to confirm block (Maybe node is starting)"
)
try:
if prev_block and prev_block.header.reps_hash and unconfirmed_header.height > 1:
prev_reps = reps_getter(prev_block.header.reps_hash)
block_verifier.verify_prev_votes(unconfirmed_block, prev_reps)
except Exception as e:
util.logger.warning(e)
traceback.print_exc()
raise ConfirmInfoInvalid(
"Unconfirmed block has no valid confirm info for previous block"
)
def __add_block_by_sync(self, block_, confirm_info=None):
util.logger.debug(
f"__add_block_by_sync :: height({block_.header.height}) hash({block_.header.hash})"
)
block_version = self.blockchain.block_versioner.get_version(
block_.header.height)
block_verifier = BlockVerifier.new(block_version,
self.blockchain.tx_versioner,
raise_exceptions=False)
block_verifier.invoke_func = self.blockchain.get_invoke_func(
block_.header.height)
reps_getter = self.blockchain.find_preps_addresses_by_roothash
block_verifier.verify_loosely(block_,
self.blockchain.last_block,
self.blockchain,
reps_getter=reps_getter)
need_to_write_tx_info, need_to_score_invoke = True, True
for exc in block_verifier.exceptions:
if isinstance(exc, TransactionDuplicatedHashError):
need_to_write_tx_info = False
if isinstance(exc, ScoreInvokeError) and not need_to_write_tx_info:
need_to_score_invoke = False
exc = next((exc for exc in block_verifier.exceptions
if not isinstance(exc, TransactionDuplicatedHashError)),
None)
if exc:
if isinstance(exc, ScoreInvokeError) and not need_to_score_invoke:
pass
else:
raise exc
if parse_version(block_.header.version) >= parse_version("0.3"):
reps = reps_getter(block_.header.reps_hash)
round_ = next(vote for vote in confirm_info if vote).round
votes = Votes.get_block_votes_class(block_.header.version)(
reps, conf.VOTING_RATIO, block_.header.height, round_,
block_.header.hash, confirm_info)
votes.verify()
return self.blockchain.add_block(block_, confirm_info,
need_to_write_tx_info,
need_to_score_invoke)
def __confirm_prev_block_by_sync(self, block_):
prev_block = self.blockchain.last_unconfirmed_block
confirm_info = block_.body.confirm_prev_block
util.logger.debug(
f"confirm_prev_block_by_sync :: height({prev_block.header.height})"
)
block_version = self.blockchain.block_versioner.get_version(
prev_block.header.height)
block_verifier = BlockVerifier.new(block_version,
self.blockchain.tx_versioner)
block_verifier.invoke_func = self.blockchain.get_invoke_func(
prev_block.header.height)
reps_getter = self.blockchain.find_preps_addresses_by_roothash
block_verifier.verify_loosely(prev_block,
self.blockchain.last_block,
self.blockchain,
reps_getter=reps_getter)
return self.blockchain.add_block(prev_block, confirm_info)
def _add_block_by_sync(self, block_, confirm_info: Optional[List] = None):
"""
TODO : If possible, change _add_block_by_sync to coroutine
:param block_:
:param confirm_info:
:return:
"""
utils.logger.debug(
f"height({block_.header.height}) hash({block_.header.hash})")
block_version = self._blockchain.block_versioner.get_version(
block_.header.height)
block_verifier = BlockVerifier.new(block_version,
self._blockchain.tx_versioner,
raise_exceptions=False)
block_verifier.invoke_func = self._blockchain.get_invoke_func(
block_.header.height)
reps_getter = self._blockchain.find_preps_addresses_by_roothash
block_verifier.verify_loosely(block_,
self._blockchain.last_block,
self._blockchain,
reps_getter=reps_getter)
need_to_write_tx_info, need_to_score_invoke = True, True
for exc in block_verifier.exceptions:
if isinstance(exc, exception.TransactionDuplicatedHashError):
need_to_write_tx_info = False
if isinstance(
exc,
exception.ScoreInvokeError) and not need_to_write_tx_info:
need_to_score_invoke = False
exc = next(
(exc for exc in block_verifier.exceptions
if not isinstance(exc, exception.TransactionDuplicatedHashError)),
None)
if exc:
if isinstance(
exc,
exception.ScoreInvokeError) and not need_to_score_invoke:
pass
else:
raise exc
try:
if parse_version(block_.header.version) >= parse_version("0.3"):
reps = reps_getter(block_.header.reps_hash)
round_ = next(vote for vote in confirm_info if vote).round
votes = Votes.get_block_votes_class(block_.header.version)(
reps, conf.VOTING_RATIO, block_.header.height, round_,
block_.header.hash, confirm_info)
votes.verify()
except StopIteration:
# I think is is unconfirmed_block because no confirm_info.
# should be fix prevent unconfirmed block later.
utils.logger.warning(f"block: {block_}")
raise Exception(f"confirm_info is empty: {confirm_info}")
self._blockchain.add_block(block_, confirm_info, need_to_write_tx_info,
need_to_score_invoke)
def test_block_v0_4(self):
block_version = "0.4"
test_signer = Signer.from_prikey(os.urandom(32))
tx_versioner = TransactionVersioner()
dummy_receipts = {}
block_builder = BlockBuilder.new(block_version, tx_versioner)
for i in range(5):
tx_builder = TransactionBuilder.new("0x3", None, tx_versioner)
tx_builder.signer = test_signer
tx_builder.to_address = ExternalAddress.new()
tx_builder.step_limit = random.randint(0, 10000)
tx_builder.value = random.randint(0, 10000)
tx_builder.nid = 2
tx = tx_builder.build()
tx_serializer = TransactionSerializer.new(tx.version, tx.type(),
tx_versioner)
block_builder.transactions[tx.hash] = tx
dummy_receipts[tx.hash.hex()] = {
"dummy_receipt": "dummy",
"tx_dumped": tx_serializer.to_full_data(tx)
}
next_leader = ExternalAddress.fromhex(
"hx00112233445566778899aabbccddeeff00112233")
block_builder.signer = test_signer
block_builder.height = 0
block_builder.prev_hash = Hash32(bytes(Hash32.size))
block_builder.state_hash = Hash32(bytes(Hash32.size))
block_builder.receipts = dummy_receipts
block_builder.reps = [
ExternalAddress.fromhex_address(test_signer.address)
]
block_builder.next_leader = next_leader
block_builder.next_reps = []
vote = BlockVote.new(test_signer, utils.get_time_stamp(),
block_builder.height - 1, 0,
block_builder.prev_hash)
votes = BlockVotes(block_builder.reps, conf.VOTING_RATIO,
block_builder.height - 1, 0,
block_builder.prev_hash)
votes.add_vote(vote)
block_builder.prev_votes = votes.votes
block = block_builder.build()
block_verifier = BlockVerifier.new(block_version, tx_versioner)
block_verifier.invoke_func = lambda b, prev_b: (block, dummy_receipts)
reps_getter = lambda _: block_builder.reps
generator = ExternalAddress.fromhex_address(test_signer.address)
block_verifier.verify(block,
None,
None,
generator=generator,
reps_getter=reps_getter)
block_serializer = BlockSerializer.new(block_version, tx_versioner)
block_serialized = block_serializer.serialize(block)
logging.info(json.dumps(block_serialized, indent=4))
block_deserialized = block_serializer.deserialize(block_serialized)
logging.info(
json.dumps(block_serializer.serialize(block_deserialized),
indent=4))
assert block.header == block_deserialized.header
assert block.body == block_deserialized.body
tx_hashes = list(block.body.transactions)
tx_index = random.randrange(0, len(tx_hashes))
block_prover = BlockProver.new(block.header.version, tx_hashes,
BlockProverType.Transaction)
tx_proof = block_prover.get_proof(tx_index)
assert block_prover.prove(tx_hashes[tx_index],
block.header.transactions_hash, tx_proof)
block_prover = BlockProver.new(block.header.version,
block_builder.receipts,
BlockProverType.Receipt)
receipt_proof = block_prover.get_proof(tx_index)
receipts_hash = block_prover.to_hash32(
block_builder.receipts[tx_index])
assert block_prover.prove(receipts_hash, block.header.receipts_hash,
receipt_proof)
