def _calculate_root_block_coinbase(self, m_hash_list: List[bytes], height: int) -> Dict: """ assumes all minor blocks in m_hash_list have been processed by slaves and thus available when looking up """ assert all([ self.db.contain_minor_block_by_hash(m_hash) for m_hash in m_hash_list ]) epoch = height // self.root_config.EPOCH_INTERVAL numerator = (self.env.quark_chain_config.block_reward_decay_factor. numerator**epoch) denominator = (self.env.quark_chain_config.block_reward_decay_factor. denominator**epoch) coinbase_amount = self.root_config.COINBASE_AMOUNT * numerator // denominator reward_tax_rate = self.env.quark_chain_config.reward_tax_rate # the ratio of minor block coinbase ratio = (1 - reward_tax_rate) / reward_tax_rate # type: Fraction reward_tokens_map = TokenBalanceMap({}) for m_hash in m_hash_list: reward_tokens_map.add( self.db.get_minor_block_coinbase_tokens(m_hash)) reward_tokens = reward_tokens_map.balance_map # note the minor block fee is after tax reward_tokens = { k: v * ratio.denominator // ratio.numerator for k, v in reward_tokens.items() } genesis_token = self.env.quark_chain_config.genesis_token reward_tokens[genesis_token] = (reward_tokens.get(genesis_token, 0) + coinbase_amount) return reward_tokens
def create_minor_block(self, root_block: RootBlock, full_shard_id: int, evm_state: EvmState) -> MinorBlock: """ Create genesis block for shard. Genesis block's hash_prev_root_block is set to the genesis root block. Genesis state will be committed to the given evm_state. Based on ALLOC, genesis_token will be added to initial accounts. """ branch = Branch(full_shard_id) shard_config = self._qkc_config.shards[full_shard_id] genesis = shard_config.GENESIS for address_hex, alloc_amount in genesis.ALLOC.items(): address = Address.create_from(bytes.fromhex(address_hex)) check( self._qkc_config.get_full_shard_id_by_full_shard_key( address.full_shard_key) == full_shard_id) evm_state.full_shard_key = address.full_shard_key if isinstance(alloc_amount, dict): for k, v in alloc_amount.items(): evm_state.delta_token_balance(address.recipient, token_id_encode(k), v) else: evm_state.delta_token_balance(address.recipient, self._qkc_config.genesis_token, alloc_amount) evm_state.commit() meta = MinorBlockMeta( hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT), hash_evm_state_root=evm_state.trie.root_hash, ) local_fee_rate = 1 - self._qkc_config.reward_tax_rate # type: Fraction coinbase_tokens = { self._qkc_config.genesis_token: shard_config.COINBASE_AMOUNT * local_fee_rate.numerator // local_fee_rate.denominator } coinbase_address = Address.create_empty_account(full_shard_id) header = MinorBlockHeader( version=genesis.VERSION, height=genesis.HEIGHT, branch=branch, hash_prev_minor_block=bytes.fromhex(genesis.HASH_PREV_MINOR_BLOCK), hash_prev_root_block=root_block.header.get_hash(), evm_gas_limit=genesis.GAS_LIMIT, hash_meta=sha3_256(meta.serialize()), coinbase_amount_map=TokenBalanceMap(coinbase_tokens), coinbase_address=coinbase_address, create_time=genesis.TIMESTAMP, difficulty=genesis.DIFFICULTY, extra_data=bytes.fromhex(genesis.EXTRA_DATA), ) return ( MinorBlock(header=header, meta=meta, tx_list=[]), TokenBalanceMap(coinbase_tokens), )
def test_add_minor_block_request_list(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) with ClusterContext(2, acc1) as clusters: shard_state = clusters[0].get_shard_state(0b10) coinbase_amount = (shard_state.env.quark_chain_config.shards[ shard_state.full_shard_id].COINBASE_AMOUNT // 2) b1 = shard_state.get_tip().create_block_to_append() evm_state = shard_state.run_block(b1) coinbase_amount_map = TokenBalanceMap(evm_state.block_fee_tokens) coinbase_amount_map.add({ shard_state.env.quark_chain_config.genesis_token: coinbase_amount }) b1.finalize(evm_state=evm_state, coinbase_amount_map=coinbase_amount_map) add_result = call_async(clusters[0].master.add_raw_minor_block( b1.header.branch, b1.serialize())) self.assertTrue(add_result) # Make sure the xshard list is not broadcasted to the other shard self.assertFalse(clusters[0].get_shard_state( 0b11).contain_remote_minor_block_hash(b1.header.get_hash())) self.assertTrue( clusters[0].master.root_state.is_minor_block_validated( b1.header.get_hash())) # Make sure another cluster received the new block assert_true_with_timeout(lambda: clusters[0].get_shard_state( 0b10).contain_block_by_hash(b1.header.get_hash())) assert_true_with_timeout( lambda: clusters[1].master.root_state.is_minor_block_validated( b1.header.get_hash()))
def _tip_gen(shard_state): coinbase_amount = (shard_state.env.quark_chain_config.shards[ shard_state.full_shard_id].COINBASE_AMOUNT // 2) b = shard_state.get_tip().create_block_to_append() evm_state = shard_state.run_block(b) coinbase_amount_map = TokenBalanceMap(evm_state.block_fee_tokens) coinbase_amount_map.add( {shard_state.env.quark_chain_config.genesis_token: coinbase_amount}) b.finalize(evm_state=evm_state, coinbase_amount_map=coinbase_amount_map) return b
def __recover_from_db(self): """ Recover the best chain from local database. """ Logger.info("Recovering root chain from local database...") if b"tipHash" not in self.db: return None r_hash = self.db.get(b"tipHash") r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash)) if r_block.header.height <= 0: return None # use the parent of the tipHash block as the new tip # since it's guaranteed to have been accepted by all the shards # while shards might not have seen the block of tipHash r_hash = r_block.header.hash_prev_block r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash)) self.tip_header = r_block.header # type: RootBlockHeader while len(self.r_header_pool) < self.max_num_blocks_to_recover: self.r_header_pool[r_hash] = r_block.header for m_header in r_block.minor_block_header_list: mtokens = TokenBalanceMap.deserialize( self.db.get(b"mheader_" + m_header.get_hash())).balance_map self.m_hash_dict[m_header.get_hash()] = mtokens if r_block.header.height <= 0: break r_hash = r_block.header.hash_prev_block r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash))
def test_add(self): m0 = TokenBalanceMap({0: 10}) m1 = TokenBalanceMap({1: 20}) m0.add(m1.balance_map) self.assertEqual(m0.balance_map, {0: 10, 1: 20}) m2 = TokenBalanceMap({0: 30, 2: 50}) m0.add(m2.balance_map) self.assertEqual(m0.balance_map, {0: 40, 1: 20, 2: 50})
def contain_minor_block_by_hash(self, h): if h in self.m_hash_dict: return True tokens = self.db.get(b"mheader_" + h) if tokens is None: return False self.m_hash_dict[h] = TokenBalanceMap.deserialize(tokens).balance_map return True
def test_token_serialization(self): # ignore 0 values in TokenBalanceMap m0 = TokenBalanceMap({3234: 10, 0: 0, 3567: 0}) m1 = TokenBalanceMap({3234: 10}) self.assertEqual(m0.serialize(bytearray()), m1.serialize(bytearray())) self.assertEqual( TokenBalanceMap.deserialize(m0.serialize(bytearray())).balance_map, {3234: 10}, ) mx = FakeTokenBalanceMap({3232: 109, 0: 0, 3567: 999999}) bb = bytearray(b"\x00\x00\x00\x03\x00\x00\x02\x0c\xa0\x01m\x02\r\xef\x03\x0fB?") self.assertEqual(mx.serialize(bytearray()), bb) self.assertEqual( TokenBalanceMap.deserialize(bb).balance_map, {3232: 109, 3567: 999999} ) # if skip_func == None, do not omit key/value pairs md0 = MapData({5: 0, 1: 2, 10: 9}) md1 = MapData({10: 9, 3: 0, 1: 2}) self.assertNotEqual(md0.serialize(), md1.serialize())
def get_account_data( self, address: Address, block_height: Optional[int]) -> List[AccountBranchData]: results = [] for branch, shard in self.shards.items(): token_balances = shard.state.get_balances(address.recipient, block_height) results.append( AccountBranchData( branch=branch, transaction_count=shard.state.get_transaction_count( address.recipient, block_height), token_balances=TokenBalanceMap(token_balances), is_contract=len( shard.state.get_code(address.recipient, block_height)) > 0, )) return results
def put_minor_block_coinbase(self, m_hash: bytes, coinbase_tokens: dict): tokens = TokenBalanceMap(coinbase_tokens) self.db.put(b"mheader_" + m_hash, tokens.serialize()) self.m_hash_dict[m_hash] = coinbase_tokens
def get_minor_block_coinbase_tokens(self, h: bytes): tokens = self.db.get(b"mheader_" + h) if tokens is None: raise KeyError() return TokenBalanceMap.deserialize(tokens).balance_map
def create_minor_block( self, root_block: RootBlock, full_shard_id: int, evm_state: EvmState) -> Tuple[MinorBlock, TokenBalanceMap]: """ Create genesis block for shard. Genesis block's hash_prev_root_block is set to the genesis root block. Genesis state will be committed to the given evm_state. Based on ALLOC, genesis_token will be added to initial accounts. """ branch = Branch(full_shard_id) shard_config = self._qkc_config.shards[full_shard_id] genesis = shard_config.GENESIS for address_hex, alloc_data in genesis.ALLOC.items(): address = Address.create_from(bytes.fromhex(address_hex)) check( self._qkc_config.get_full_shard_id_by_full_shard_key( address.full_shard_key) == full_shard_id) evm_state.full_shard_key = address.full_shard_key recipient = address.recipient if "code" in alloc_data: code = decode_hex(remove_0x_head(alloc_data["code"])) evm_state.set_code(recipient, code) evm_state.set_nonce(recipient, 1) if "storage" in alloc_data: for k, v in alloc_data["storage"].items(): evm_state.set_storage_data( recipient, big_endian_to_int(decode_hex(k[2:])), big_endian_to_int(decode_hex(v[2:])), ) # backward compatible: # v1: {addr: {QKC: 1234}} # v2: {addr: {balances: {QKC: 1234}, code: 0x, storage: {0x12: 0x34}}} balances = alloc_data if "balances" in alloc_data: balances = alloc_data["balances"] for k, v in balances.items(): if k in ("code", "storage"): continue evm_state.delta_token_balance(recipient, token_id_encode(k), v) evm_state.commit() meta = MinorBlockMeta( hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT), hash_evm_state_root=evm_state.trie.root_hash, xshard_tx_cursor_info=XshardTxCursorInfo(root_block.header.height, 0, 0), ) local_fee_rate = 1 - self._qkc_config.reward_tax_rate # type: Fraction coinbase_tokens = { self._qkc_config.genesis_token: shard_config.COINBASE_AMOUNT * local_fee_rate.numerator // local_fee_rate.denominator } coinbase_address = Address.create_empty_account(full_shard_id) header = MinorBlockHeader( version=genesis.VERSION, height=genesis.HEIGHT, branch=branch, hash_prev_minor_block=bytes.fromhex(genesis.HASH_PREV_MINOR_BLOCK), hash_prev_root_block=root_block.header.get_hash(), evm_gas_limit=genesis.GAS_LIMIT, hash_meta=sha3_256(meta.serialize()), coinbase_amount_map=TokenBalanceMap(coinbase_tokens), coinbase_address=coinbase_address, create_time=genesis.TIMESTAMP, difficulty=genesis.DIFFICULTY, extra_data=bytes.fromhex(genesis.EXTRA_DATA), ) return ( MinorBlock(header=header, meta=meta, tx_list=[]), TokenBalanceMap(coinbase_tokens), )
def test_zero_balance(self): m0 = TokenBalanceMap({3234: 10, 0: 0, 3567: 0}) m1 = TokenBalanceMap.deserialize(m0.serialize()) self.assertEqual(m0, m1)
def test_shard_synchronizer_with_fork(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) with ClusterContext(2, acc1) as clusters: # shutdown cluster connection clusters[1].peer.close() block_list = [] # cluster 0 has 13 blocks added shard_state0 = clusters[0].get_shard_state(0b10) coinbase_amount = (shard_state0.env.quark_chain_config.shards[ shard_state0.full_shard_id].COINBASE_AMOUNT // 2) for i in range(13): block = shard_state0.get_tip().create_block_to_append() evm_state = shard_state0.run_block(block) coinbase_amount_map = TokenBalanceMap( evm_state.block_fee_tokens) coinbase_amount_map.add({ shard_state0.env.quark_chain_config.genesis_token: coinbase_amount }) block.finalize(evm_state=evm_state, coinbase_amount_map=coinbase_amount_map) add_result = call_async(clusters[0].master.add_raw_minor_block( block.header.branch, block.serialize())) self.assertTrue(add_result) block_list.append(block) self.assertEqual( clusters[0].get_shard_state(0b10).header_tip.height, 13) # cluster 1 has 12 blocks added shard_state0 = clusters[1].get_shard_state(0b10) coinbase_amount = (shard_state0.env.quark_chain_config.shards[ shard_state0.full_shard_id].COINBASE_AMOUNT // 2) for i in range(12): block = shard_state0.get_tip().create_block_to_append() evm_state = shard_state0.run_block(block) coinbase_amount_map = TokenBalanceMap( evm_state.block_fee_tokens) coinbase_amount_map.add({ shard_state0.env.quark_chain_config.genesis_token: coinbase_amount }) block.finalize(evm_state=evm_state, coinbase_amount_map=coinbase_amount_map) add_result = call_async(clusters[1].master.add_raw_minor_block( block.header.branch, block.serialize())) self.assertTrue(add_result) self.assertEqual( clusters[1].get_shard_state(0b10).header_tip.height, 12) # reestablish cluster connection call_async(clusters[1].network.connect( "127.0.0.1", clusters[0].master.env.cluster_config.SIMPLE_NETWORK. BOOTSTRAP_PORT, )) # a new block from cluster 0 will trigger sync in cluster 1 shard_state0 = clusters[0].get_shard_state(0b10) coinbase_amount = (shard_state0.env.quark_chain_config.shards[ shard_state0.full_shard_id].COINBASE_AMOUNT // 2) block = shard_state0.get_tip().create_block_to_append() evm_state = shard_state0.run_block(block) coinbase_amount_map = TokenBalanceMap(evm_state.block_fee_tokens) coinbase_amount_map.add({ shard_state0.env.quark_chain_config.genesis_token: coinbase_amount }) block.finalize(evm_state=evm_state, coinbase_amount_map=coinbase_amount_map) add_result = call_async(clusters[0].master.add_raw_minor_block( block.header.branch, block.serialize())) self.assertTrue(add_result) block_list.append(block) # expect cluster 1 has all the blocks from cluter 0 and # has the same tip as cluster 0 for block in block_list: assert_true_with_timeout( lambda: clusters[1].slave_list[0].shards[Branch(0b10)]. state.contain_block_by_hash(block.header.get_hash())) assert_true_with_timeout( lambda: clusters[1].master.root_state. is_minor_block_validated(block.header.get_hash())) self.assertEqual( clusters[1].get_shard_state(0b10).header_tip, clusters[0].get_shard_state(0b10).header_tip, )
def test_add_root_block_request_list(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) with ClusterContext(2, acc1) as clusters: # shutdown cluster connection clusters[1].peer.close() # add blocks in cluster 0 block_header_list = [clusters[0].get_shard_state(2 | 0).header_tip] shard_state0 = clusters[0].get_shard_state(0b10) coinbase_amount = (shard_state0.env.quark_chain_config.shards[ shard_state0.full_shard_id].COINBASE_AMOUNT // 2) for i in range(7): b1 = shard_state0.get_tip().create_block_to_append() evm_state = shard_state0.run_block(b1) coinbase_amount_map = TokenBalanceMap( evm_state.block_fee_tokens) coinbase_amount_map.add({ shard_state0.env.quark_chain_config.genesis_token: coinbase_amount }) b1.finalize(evm_state=evm_state, coinbase_amount_map=coinbase_amount_map) add_result = call_async(clusters[0].master.add_raw_minor_block( b1.header.branch, b1.serialize())) self.assertTrue(add_result) block_header_list.append(b1.header) block_header_list.append( clusters[0].get_shard_state(2 | 1).header_tip) shard_state0 = clusters[0].get_shard_state(0b11) coinbase_amount = (shard_state0.env.quark_chain_config.shards[ shard_state0.full_shard_id].COINBASE_AMOUNT // 2) b2 = shard_state0.get_tip().create_block_to_append() evm_state = shard_state0.run_block(b2) coinbase_amount_map = TokenBalanceMap(evm_state.block_fee_tokens) coinbase_amount_map.add({ shard_state0.env.quark_chain_config.genesis_token: coinbase_amount }) b2.finalize(evm_state=evm_state, coinbase_amount_map=coinbase_amount_map) add_result = call_async(clusters[0].master.add_raw_minor_block( b2.header.branch, b2.serialize())) self.assertTrue(add_result) block_header_list.append(b2.header) # add 1 block in cluster 1 shard_state1 = clusters[1].get_shard_state(0b11) coinbase_amount = (shard_state1.env.quark_chain_config.shards[ shard_state1.full_shard_id].COINBASE_AMOUNT // 2) b3 = shard_state1.get_tip().create_block_to_append() evm_state = shard_state1.run_block(b3) coinbase_amount_map = TokenBalanceMap(evm_state.block_fee_tokens) coinbase_amount_map.add({ shard_state1.env.quark_chain_config.genesis_token: coinbase_amount }) b3.finalize(evm_state=evm_state, coinbase_amount_map=coinbase_amount_map) add_result = call_async(clusters[1].master.add_raw_minor_block( b3.header.branch, b3.serialize())) self.assertTrue(add_result) self.assertEqual(clusters[1].get_shard_state(0b11).header_tip, b3.header) # reestablish cluster connection call_async(clusters[1].network.connect( "127.0.0.1", clusters[0].master.env.cluster_config.SIMPLE_NETWORK. BOOTSTRAP_PORT, )) root_block1 = clusters[0].master.root_state.create_block_to_mine( block_header_list, acc1) call_async(clusters[0].master.add_root_block(root_block1)) # Make sure the root block tip of local cluster is changed self.assertEqual(clusters[0].master.root_state.tip, root_block1.header) # Make sure the root block tip of cluster 1 is changed assert_true_with_timeout( lambda: clusters[1].master.root_state.tip == root_block1. header, 2) # Minor block is downloaded self.assertEqual(b1.header.height, 7) assert_true_with_timeout(lambda: clusters[1].get_shard_state(0b10). header_tip == b1.header) # The tip is overwritten due to root chain first consensus assert_true_with_timeout(lambda: clusters[1].get_shard_state(0b11). header_tip == b2.header)