def _calculate_root_block_coinbase(self, m_hash_list: List[bytes],
height: int) -> Dict:
"""
assumes all minor blocks in m_hash_list have been processed by slaves and thus available when looking up
"""
assert all([
self.db.contain_minor_block_by_hash(m_hash)
for m_hash in m_hash_list
])
epoch = height // self.root_config.EPOCH_INTERVAL
numerator = (self.env.quark_chain_config.block_reward_decay_factor.
numerator**epoch)
denominator = (self.env.quark_chain_config.block_reward_decay_factor.
denominator**epoch)
coinbase_amount = self.root_config.COINBASE_AMOUNT * numerator // denominator
reward_tax_rate = self.env.quark_chain_config.reward_tax_rate
# the ratio of minor block coinbase
ratio = (1 - reward_tax_rate) / reward_tax_rate # type: Fraction
reward_tokens_map = TokenBalanceMap({})
for m_hash in m_hash_list:
reward_tokens_map.add(
self.db.get_minor_block_coinbase_tokens(m_hash))
reward_tokens = reward_tokens_map.balance_map
# note the minor block fee is after tax
reward_tokens = {
k: v * ratio.denominator // ratio.numerator
for k, v in reward_tokens.items()
}
genesis_token = self.env.quark_chain_config.genesis_token
reward_tokens[genesis_token] = (reward_tokens.get(genesis_token, 0) +
coinbase_amount)
return reward_tokens
def create_minor_block(self, root_block: RootBlock, full_shard_id: int,
evm_state: EvmState) -> MinorBlock:
""" Create genesis block for shard.
Genesis block's hash_prev_root_block is set to the genesis root block.
Genesis state will be committed to the given evm_state.
Based on ALLOC, genesis_token will be added to initial accounts.
"""
branch = Branch(full_shard_id)
shard_config = self._qkc_config.shards[full_shard_id]
genesis = shard_config.GENESIS
for address_hex, alloc_amount in genesis.ALLOC.items():
address = Address.create_from(bytes.fromhex(address_hex))
check(
self._qkc_config.get_full_shard_id_by_full_shard_key(
address.full_shard_key) == full_shard_id)
evm_state.full_shard_key = address.full_shard_key
if isinstance(alloc_amount, dict):
for k, v in alloc_amount.items():
evm_state.delta_token_balance(address.recipient,
token_id_encode(k), v)
else:
evm_state.delta_token_balance(address.recipient,
self._qkc_config.genesis_token,
alloc_amount)
evm_state.commit()
meta = MinorBlockMeta(
hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT),
hash_evm_state_root=evm_state.trie.root_hash,
)
local_fee_rate = 1 - self._qkc_config.reward_tax_rate # type: Fraction
coinbase_tokens = {
self._qkc_config.genesis_token:
shard_config.COINBASE_AMOUNT * local_fee_rate.numerator //
local_fee_rate.denominator
}
coinbase_address = Address.create_empty_account(full_shard_id)
header = MinorBlockHeader(
version=genesis.VERSION,
height=genesis.HEIGHT,
branch=branch,
hash_prev_minor_block=bytes.fromhex(genesis.HASH_PREV_MINOR_BLOCK),
hash_prev_root_block=root_block.header.get_hash(),
evm_gas_limit=genesis.GAS_LIMIT,
hash_meta=sha3_256(meta.serialize()),
coinbase_amount_map=TokenBalanceMap(coinbase_tokens),
coinbase_address=coinbase_address,
create_time=genesis.TIMESTAMP,
difficulty=genesis.DIFFICULTY,
extra_data=bytes.fromhex(genesis.EXTRA_DATA),
)
return (
MinorBlock(header=header, meta=meta, tx_list=[]),
TokenBalanceMap(coinbase_tokens),
)
def test_add_minor_block_request_list(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_key=0)
with ClusterContext(2, acc1) as clusters:
shard_state = clusters[0].get_shard_state(0b10)
coinbase_amount = (shard_state.env.quark_chain_config.shards[
shard_state.full_shard_id].COINBASE_AMOUNT // 2)
b1 = shard_state.get_tip().create_block_to_append()
evm_state = shard_state.run_block(b1)
coinbase_amount_map = TokenBalanceMap(evm_state.block_fee_tokens)
coinbase_amount_map.add({
shard_state.env.quark_chain_config.genesis_token:
coinbase_amount
})
b1.finalize(evm_state=evm_state,
coinbase_amount_map=coinbase_amount_map)
add_result = call_async(clusters[0].master.add_raw_minor_block(
b1.header.branch, b1.serialize()))
self.assertTrue(add_result)
# Make sure the xshard list is not broadcasted to the other shard
self.assertFalse(clusters[0].get_shard_state(
0b11).contain_remote_minor_block_hash(b1.header.get_hash()))
self.assertTrue(
clusters[0].master.root_state.is_minor_block_validated(
b1.header.get_hash()))
# Make sure another cluster received the new block
assert_true_with_timeout(lambda: clusters[0].get_shard_state(
0b10).contain_block_by_hash(b1.header.get_hash()))
assert_true_with_timeout(
lambda: clusters[1].master.root_state.is_minor_block_validated(
b1.header.get_hash()))
def _tip_gen(shard_state):
coinbase_amount = (shard_state.env.quark_chain_config.shards[
shard_state.full_shard_id].COINBASE_AMOUNT // 2)
b = shard_state.get_tip().create_block_to_append()
evm_state = shard_state.run_block(b)
coinbase_amount_map = TokenBalanceMap(evm_state.block_fee_tokens)
coinbase_amount_map.add(
{shard_state.env.quark_chain_config.genesis_token: coinbase_amount})
b.finalize(evm_state=evm_state, coinbase_amount_map=coinbase_amount_map)
return b
def __recover_from_db(self):
""" Recover the best chain from local database.
"""
Logger.info("Recovering root chain from local database...")
if b"tipHash" not in self.db:
return None
r_hash = self.db.get(b"tipHash")
r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash))
if r_block.header.height <= 0:
return None
# use the parent of the tipHash block as the new tip
# since it's guaranteed to have been accepted by all the shards
# while shards might not have seen the block of tipHash
r_hash = r_block.header.hash_prev_block
r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash))
self.tip_header = r_block.header # type: RootBlockHeader
while len(self.r_header_pool) < self.max_num_blocks_to_recover:
self.r_header_pool[r_hash] = r_block.header
for m_header in r_block.minor_block_header_list:
mtokens = TokenBalanceMap.deserialize(
self.db.get(b"mheader_" + m_header.get_hash())).balance_map
self.m_hash_dict[m_header.get_hash()] = mtokens
if r_block.header.height <= 0:
break
r_hash = r_block.header.hash_prev_block
r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash))
def test_add(self):
m0 = TokenBalanceMap({0: 10})
m1 = TokenBalanceMap({1: 20})
m0.add(m1.balance_map)
self.assertEqual(m0.balance_map, {0: 10, 1: 20})
m2 = TokenBalanceMap({0: 30, 2: 50})
m0.add(m2.balance_map)
self.assertEqual(m0.balance_map, {0: 40, 1: 20, 2: 50})
def contain_minor_block_by_hash(self, h):
if h in self.m_hash_dict:
return True
tokens = self.db.get(b"mheader_" + h)
if tokens is None:
return False
self.m_hash_dict[h] = TokenBalanceMap.deserialize(tokens).balance_map
return True
def test_token_serialization(self):
# ignore 0 values in TokenBalanceMap
m0 = TokenBalanceMap({3234: 10, 0: 0, 3567: 0})
m1 = TokenBalanceMap({3234: 10})
self.assertEqual(m0.serialize(bytearray()), m1.serialize(bytearray()))
self.assertEqual(
TokenBalanceMap.deserialize(m0.serialize(bytearray())).balance_map,
{3234: 10},
)
mx = FakeTokenBalanceMap({3232: 109, 0: 0, 3567: 999999})
bb = bytearray(b"\x00\x00\x00\x03\x00\x00\x02\x0c\xa0\x01m\x02\r\xef\x03\x0fB?")
self.assertEqual(mx.serialize(bytearray()), bb)
self.assertEqual(
TokenBalanceMap.deserialize(bb).balance_map, {3232: 109, 3567: 999999}
)
# if skip_func == None, do not omit key/value pairs
md0 = MapData({5: 0, 1: 2, 10: 9})
md1 = MapData({10: 9, 3: 0, 1: 2})
self.assertNotEqual(md0.serialize(), md1.serialize())
def get_account_data(
self, address: Address,
block_height: Optional[int]) -> List[AccountBranchData]:
results = []
for branch, shard in self.shards.items():
token_balances = shard.state.get_balances(address.recipient,
block_height)
results.append(
AccountBranchData(
branch=branch,
transaction_count=shard.state.get_transaction_count(
address.recipient, block_height),
token_balances=TokenBalanceMap(token_balances),
is_contract=len(
shard.state.get_code(address.recipient, block_height))
> 0,
))
return results
def put_minor_block_coinbase(self, m_hash: bytes, coinbase_tokens: dict):
tokens = TokenBalanceMap(coinbase_tokens)
self.db.put(b"mheader_" + m_hash, tokens.serialize())
self.m_hash_dict[m_hash] = coinbase_tokens
def get_minor_block_coinbase_tokens(self, h: bytes):
tokens = self.db.get(b"mheader_" + h)
if tokens is None:
raise KeyError()
return TokenBalanceMap.deserialize(tokens).balance_map
def create_minor_block(
self, root_block: RootBlock, full_shard_id: int,
evm_state: EvmState) -> Tuple[MinorBlock, TokenBalanceMap]:
""" Create genesis block for shard.
Genesis block's hash_prev_root_block is set to the genesis root block.
Genesis state will be committed to the given evm_state.
Based on ALLOC, genesis_token will be added to initial accounts.
"""
branch = Branch(full_shard_id)
shard_config = self._qkc_config.shards[full_shard_id]
genesis = shard_config.GENESIS
for address_hex, alloc_data in genesis.ALLOC.items():
address = Address.create_from(bytes.fromhex(address_hex))
check(
self._qkc_config.get_full_shard_id_by_full_shard_key(
address.full_shard_key) == full_shard_id)
evm_state.full_shard_key = address.full_shard_key
recipient = address.recipient
if "code" in alloc_data:
code = decode_hex(remove_0x_head(alloc_data["code"]))
evm_state.set_code(recipient, code)
evm_state.set_nonce(recipient, 1)
if "storage" in alloc_data:
for k, v in alloc_data["storage"].items():
evm_state.set_storage_data(
recipient,
big_endian_to_int(decode_hex(k[2:])),
big_endian_to_int(decode_hex(v[2:])),
)
# backward compatible:
# v1: {addr: {QKC: 1234}}
# v2: {addr: {balances: {QKC: 1234}, code: 0x, storage: {0x12: 0x34}}}
balances = alloc_data
if "balances" in alloc_data:
balances = alloc_data["balances"]
for k, v in balances.items():
if k in ("code", "storage"):
continue
evm_state.delta_token_balance(recipient, token_id_encode(k), v)
evm_state.commit()
meta = MinorBlockMeta(
hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT),
hash_evm_state_root=evm_state.trie.root_hash,
xshard_tx_cursor_info=XshardTxCursorInfo(root_block.header.height,
0, 0),
)
local_fee_rate = 1 - self._qkc_config.reward_tax_rate # type: Fraction
coinbase_tokens = {
self._qkc_config.genesis_token:
shard_config.COINBASE_AMOUNT * local_fee_rate.numerator //
local_fee_rate.denominator
}
coinbase_address = Address.create_empty_account(full_shard_id)
header = MinorBlockHeader(
version=genesis.VERSION,
height=genesis.HEIGHT,
branch=branch,
hash_prev_minor_block=bytes.fromhex(genesis.HASH_PREV_MINOR_BLOCK),
hash_prev_root_block=root_block.header.get_hash(),
evm_gas_limit=genesis.GAS_LIMIT,
hash_meta=sha3_256(meta.serialize()),
coinbase_amount_map=TokenBalanceMap(coinbase_tokens),
coinbase_address=coinbase_address,
create_time=genesis.TIMESTAMP,
difficulty=genesis.DIFFICULTY,
extra_data=bytes.fromhex(genesis.EXTRA_DATA),
)
return (
MinorBlock(header=header, meta=meta, tx_list=[]),
TokenBalanceMap(coinbase_tokens),
)
def test_zero_balance(self):
m0 = TokenBalanceMap({3234: 10, 0: 0, 3567: 0})
m1 = TokenBalanceMap.deserialize(m0.serialize())
self.assertEqual(m0, m1)
def test_shard_synchronizer_with_fork(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_key=0)
with ClusterContext(2, acc1) as clusters:
# shutdown cluster connection
clusters[1].peer.close()
block_list = []
# cluster 0 has 13 blocks added
shard_state0 = clusters[0].get_shard_state(0b10)
coinbase_amount = (shard_state0.env.quark_chain_config.shards[
shard_state0.full_shard_id].COINBASE_AMOUNT // 2)
for i in range(13):
block = shard_state0.get_tip().create_block_to_append()
evm_state = shard_state0.run_block(block)
coinbase_amount_map = TokenBalanceMap(
evm_state.block_fee_tokens)
coinbase_amount_map.add({
shard_state0.env.quark_chain_config.genesis_token:
coinbase_amount
})
block.finalize(evm_state=evm_state,
coinbase_amount_map=coinbase_amount_map)
add_result = call_async(clusters[0].master.add_raw_minor_block(
block.header.branch, block.serialize()))
self.assertTrue(add_result)
block_list.append(block)
self.assertEqual(
clusters[0].get_shard_state(0b10).header_tip.height, 13)
# cluster 1 has 12 blocks added
shard_state0 = clusters[1].get_shard_state(0b10)
coinbase_amount = (shard_state0.env.quark_chain_config.shards[
shard_state0.full_shard_id].COINBASE_AMOUNT // 2)
for i in range(12):
block = shard_state0.get_tip().create_block_to_append()
evm_state = shard_state0.run_block(block)
coinbase_amount_map = TokenBalanceMap(
evm_state.block_fee_tokens)
coinbase_amount_map.add({
shard_state0.env.quark_chain_config.genesis_token:
coinbase_amount
})
block.finalize(evm_state=evm_state,
coinbase_amount_map=coinbase_amount_map)
add_result = call_async(clusters[1].master.add_raw_minor_block(
block.header.branch, block.serialize()))
self.assertTrue(add_result)
self.assertEqual(
clusters[1].get_shard_state(0b10).header_tip.height, 12)
# reestablish cluster connection
call_async(clusters[1].network.connect(
"127.0.0.1",
clusters[0].master.env.cluster_config.SIMPLE_NETWORK.
BOOTSTRAP_PORT,
))
# a new block from cluster 0 will trigger sync in cluster 1
shard_state0 = clusters[0].get_shard_state(0b10)
coinbase_amount = (shard_state0.env.quark_chain_config.shards[
shard_state0.full_shard_id].COINBASE_AMOUNT // 2)
block = shard_state0.get_tip().create_block_to_append()
evm_state = shard_state0.run_block(block)
coinbase_amount_map = TokenBalanceMap(evm_state.block_fee_tokens)
coinbase_amount_map.add({
shard_state0.env.quark_chain_config.genesis_token:
coinbase_amount
})
block.finalize(evm_state=evm_state,
coinbase_amount_map=coinbase_amount_map)
add_result = call_async(clusters[0].master.add_raw_minor_block(
block.header.branch, block.serialize()))
self.assertTrue(add_result)
block_list.append(block)
# expect cluster 1 has all the blocks from cluter 0 and
# has the same tip as cluster 0
for block in block_list:
assert_true_with_timeout(
lambda: clusters[1].slave_list[0].shards[Branch(0b10)].
state.contain_block_by_hash(block.header.get_hash()))
assert_true_with_timeout(
lambda: clusters[1].master.root_state.
is_minor_block_validated(block.header.get_hash()))
self.assertEqual(
clusters[1].get_shard_state(0b10).header_tip,
clusters[0].get_shard_state(0b10).header_tip,
)
def test_add_root_block_request_list(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_key=0)
with ClusterContext(2, acc1) as clusters:
# shutdown cluster connection
clusters[1].peer.close()
# add blocks in cluster 0
block_header_list = [clusters[0].get_shard_state(2 | 0).header_tip]
shard_state0 = clusters[0].get_shard_state(0b10)
coinbase_amount = (shard_state0.env.quark_chain_config.shards[
shard_state0.full_shard_id].COINBASE_AMOUNT // 2)
for i in range(7):
b1 = shard_state0.get_tip().create_block_to_append()
evm_state = shard_state0.run_block(b1)
coinbase_amount_map = TokenBalanceMap(
evm_state.block_fee_tokens)
coinbase_amount_map.add({
shard_state0.env.quark_chain_config.genesis_token:
coinbase_amount
})
b1.finalize(evm_state=evm_state,
coinbase_amount_map=coinbase_amount_map)
add_result = call_async(clusters[0].master.add_raw_minor_block(
b1.header.branch, b1.serialize()))
self.assertTrue(add_result)
block_header_list.append(b1.header)
block_header_list.append(
clusters[0].get_shard_state(2 | 1).header_tip)
shard_state0 = clusters[0].get_shard_state(0b11)
coinbase_amount = (shard_state0.env.quark_chain_config.shards[
shard_state0.full_shard_id].COINBASE_AMOUNT // 2)
b2 = shard_state0.get_tip().create_block_to_append()
evm_state = shard_state0.run_block(b2)
coinbase_amount_map = TokenBalanceMap(evm_state.block_fee_tokens)
coinbase_amount_map.add({
shard_state0.env.quark_chain_config.genesis_token:
coinbase_amount
})
b2.finalize(evm_state=evm_state,
coinbase_amount_map=coinbase_amount_map)
add_result = call_async(clusters[0].master.add_raw_minor_block(
b2.header.branch, b2.serialize()))
self.assertTrue(add_result)
block_header_list.append(b2.header)
# add 1 block in cluster 1
shard_state1 = clusters[1].get_shard_state(0b11)
coinbase_amount = (shard_state1.env.quark_chain_config.shards[
shard_state1.full_shard_id].COINBASE_AMOUNT // 2)
b3 = shard_state1.get_tip().create_block_to_append()
evm_state = shard_state1.run_block(b3)
coinbase_amount_map = TokenBalanceMap(evm_state.block_fee_tokens)
coinbase_amount_map.add({
shard_state1.env.quark_chain_config.genesis_token:
coinbase_amount
})
b3.finalize(evm_state=evm_state,
coinbase_amount_map=coinbase_amount_map)
add_result = call_async(clusters[1].master.add_raw_minor_block(
b3.header.branch, b3.serialize()))
self.assertTrue(add_result)
self.assertEqual(clusters[1].get_shard_state(0b11).header_tip,
b3.header)
# reestablish cluster connection
call_async(clusters[1].network.connect(
"127.0.0.1",
clusters[0].master.env.cluster_config.SIMPLE_NETWORK.
BOOTSTRAP_PORT,
))
root_block1 = clusters[0].master.root_state.create_block_to_mine(
block_header_list, acc1)
call_async(clusters[0].master.add_root_block(root_block1))
# Make sure the root block tip of local cluster is changed
self.assertEqual(clusters[0].master.root_state.tip,
root_block1.header)
# Make sure the root block tip of cluster 1 is changed
assert_true_with_timeout(
lambda: clusters[1].master.root_state.tip == root_block1.
header, 2)
# Minor block is downloaded
self.assertEqual(b1.header.height, 7)
assert_true_with_timeout(lambda: clusters[1].get_shard_state(0b10).
header_tip == b1.header)
# The tip is overwritten due to root chain first consensus
assert_true_with_timeout(lambda: clusters[1].get_shard_state(0b11).
header_tip == b2.header)
