def __get_branch_to_add_xshard_tx_list_request(self, block_hash,
xshard_tx_list,
prev_root_height):
xshard_map = dict(
) # type: Dict[Branch, List[CrossShardTransactionDeposit]]
# only broadcast to the shards that have been initialized
initialized_full_shard_ids = self.env.quark_chain_config.get_initialized_full_shard_ids_before_root_height(
prev_root_height)
for full_shard_id in initialized_full_shard_ids:
branch = Branch(full_shard_id)
xshard_map[branch] = []
for xshard_tx in xshard_tx_list:
full_shard_id = self.env.quark_chain_config.get_full_shard_id_by_full_shard_key(
xshard_tx.to_address.full_shard_key)
branch = Branch(full_shard_id)
check(branch in xshard_map)
xshard_map[branch].append(xshard_tx)
branch_to_add_xshard_tx_list_request = (
dict()) # type: Dict[Branch, AddXshardTxListRequest]
for branch, tx_list in xshard_map.items():
cross_shard_tx_list = CrossShardTransactionList(tx_list)
request = AddXshardTxListRequest(branch, block_hash,
cross_shard_tx_list)
branch_to_add_xshard_tx_list_request[branch] = request
return branch_to_add_xshard_tx_list_request
def test_add_minor_block_request_list(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_key=0)
with ClusterContext(2, acc1) as clusters:
shard_state = clusters[0].slave_list[0].shards[Branch(0b10)].state
coinbase_amount = (shard_state.env.quark_chain_config.shards[
shard_state.full_shard_id].COINBASE_AMOUNT // 2)
b1 = shard_state.get_tip().create_block_to_append()
evm_state = shard_state.run_block(b1)
b1.finalize(
evm_state=evm_state,
coinbase_amount=evm_state.block_fee + coinbase_amount,
)
add_result = call_async(clusters[0].master.add_raw_minor_block(
b1.header.branch, b1.serialize()))
self.assertTrue(add_result)
# Make sure the xshard list is not broadcasted to the other shard
self.assertFalse(clusters[0].slave_list[1].shards[Branch(
0b11)].state.contain_remote_minor_block_hash(
b1.header.get_hash()))
self.assertTrue(
clusters[0].master.root_state.is_minor_block_validated(
b1.header.get_hash()))
# Make sure another cluster received the new block
assert_true_with_timeout(
lambda: clusters[1].slave_list[0].shards[Branch(
0b10)].state.contain_block_by_hash(b1.header.get_hash()))
assert_true_with_timeout(
lambda: clusters[1].master.root_state.is_minor_block_validated(
b1.header.get_hash()))
async def _get_logs(self, data, full_shard_key, decoder: Callable[[str], bytes]):
start_block = data.get("fromBlock", "latest")
end_block = data.get("toBlock", "latest")
# TODO: not supported yet for "earliest" or "pending" block
if (isinstance(start_block, str) and start_block != "latest") or (
isinstance(end_block, str) and end_block != "latest"
):
return None
# parse addresses / topics
addresses, topics = [], []
if "address" in data:
if isinstance(data["address"], str):
addresses = [Address.deserialize(decoder(data["address"]))]
elif isinstance(data["address"], list):
addresses = [Address.deserialize(decoder(a)) for a in data["address"]]
if full_shard_key is not None:
addresses = [Address(a.recipient, full_shard_key) for a in addresses]
if "topics" in data:
for topic_item in data["topics"]:
if isinstance(topic_item, str):
topics.append([data_decoder(topic_item)])
elif isinstance(topic_item, list):
topics.append([data_decoder(tp) for tp in topic_item])
branch = Branch(
self.master.env.quark_chain_config.get_full_shard_id_by_full_shard_key(
full_shard_key
)
)
logs = await self.master.get_logs(
addresses, topics, start_block, end_block, branch
)
if logs is None:
return None
return loglist_encoder(logs)
async def eth_getBlockByNumber(self, block_height, include_transactions):
"""
NOTE: only support block_id "latest" or hex
"""
def block_transcoder(block):
"""
QuarkChain Block => ETH Block
"""
return {
**block,
"number": block["height"],
"parentHash": block["hashPrevMinorBlock"],
"sha3Uncles": "",
"logsBloom": "",
"transactionsRoot": block["hashMerkleRoot"], # ?
"stateRoot": block["hashEvmStateRoot"], # ?
}
branch = Branch(
self.master.env.quark_chain_config.
get_full_shard_id_by_full_shard_key(0))
block = await self.master.get_minor_block_by_height(
block_height, branch)
if block is None:
return None
return block_transcoder(minor_block_encoder(block))
async def create_shards(self, root_block: RootBlock):
""" Create shards based on GENESIS config and root block height if they have
not been created yet."""
async def __init_shard(shard):
await shard.init_from_root_block(root_block)
await shard.create_peer_shard_connections(self.cluster_peer_ids,
self.master)
branch = Branch(shard.full_shard_id)
self.shards[branch] = shard
if self.mining:
shard.miner.start()
new_shards = []
for (full_shard_id,
shard_config) in self.env.quark_chain_config.shards.items():
branch = Branch(full_shard_id)
if branch in self.shards:
continue
if not self.__cover_shard_id(
full_shard_id) or not shard_config.GENESIS:
continue
if root_block.header.height >= shard_config.GENESIS.ROOT_HEIGHT:
new_shards.append(Shard(self.env, full_shard_id, self))
await asyncio.gather(*[__init_shard(shard) for shard in new_shards])
def add_peer(self, peer: PeerShardConnection):
self.peers[peer.cluster_peer_id] = peer
Logger.info(
"[{}] connected to peer {}".format(
Branch(self.full_shard_id).to_str(), peer.cluster_peer_id
)
)
async def getTransactionConfirmedByNumberRootBlocks(self, tx_id):
tx_hash, full_shard_key = tx_id
branch = Branch(
self.master.env.quark_chain_config.get_full_shard_id_by_full_shard_key(
full_shard_key
)
)
minor_block, i = await self.master.get_transaction_by_hash(tx_hash, branch)
if not minor_block:
return None
if len(minor_block.tx_list) <= i:
return None
confirming_hash = self.master.root_state.db.get_root_block_confirming_minor_block(
minor_block.header.get_hash()
+ minor_block.header.branch.get_full_shard_id().to_bytes(4, byteorder="big")
)
if confirming_hash is None:
return quantity_encoder(0)
confirming_header = self.master.root_state.db.get_root_block_header_by_hash(
confirming_hash
)
canonical_hash = self.master.root_state.db.get_root_block_hash_by_height(
confirming_header.height
)
if canonical_hash != confirming_hash:
return quantity_encoder(0)
tip = self.master.root_state.tip
return quantity_encoder(tip.height - confirming_header.height + 1)
async def getTransactionConfirmedByNumberRootBlocks(self, tx_id):
tx_hash, full_shard_key = tx_id
branch = Branch(
self.master.env.quark_chain_config.
get_full_shard_id_by_full_shard_key(full_shard_key))
minor_block, i = await self.master.get_transaction_by_hash(
tx_hash, branch)
if not minor_block:
return None
if len(minor_block.tx_list) <= i:
return None
root_hash = self.master.root_state.db.get_root_block_confirming_minor_block(
minor_block.header.get_hash() +
minor_block.header.branch.get_full_shard_id().to_bytes(
4, byteorder="big"))
if root_hash is None:
return quantity_encoder(0)
root_header_tip = self.master.root_state.tip
root_header = self.master.root_state.db.get_root_block_header_by_hash(
root_hash, consistency_check=False)
if not self.master.root_state.is_same_chain(root_header_tip,
root_header):
return quantity_encoder(0)
return quantity_encoder(root_header_tip.height - root_header.height +
1)
def create_minor_block(self, root_block: RootBlock, full_shard_id: int,
evm_state: EvmState) -> MinorBlock:
""" Create genesis block for shard.
Genesis block's hash_prev_root_block is set to the genesis root block.
Genesis state will be committed to the given evm_state.
Based on ALLOC, genesis_token will be added to initial accounts.
"""
branch = Branch(full_shard_id)
shard_config = self._qkc_config.shards[full_shard_id]
genesis = shard_config.GENESIS
for address_hex, alloc_amount in genesis.ALLOC.items():
address = Address.create_from(bytes.fromhex(address_hex))
check(
self._qkc_config.get_full_shard_id_by_full_shard_key(
address.full_shard_key) == full_shard_id)
evm_state.full_shard_key = address.full_shard_key
if isinstance(alloc_amount, dict):
for k, v in alloc_amount.items():
evm_state.delta_token_balance(address.recipient,
token_id_encode(k), v)
else:
evm_state.delta_token_balance(address.recipient,
self._qkc_config.genesis_token,
alloc_amount)
evm_state.commit()
meta = MinorBlockMeta(
hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT),
hash_evm_state_root=evm_state.trie.root_hash,
)
local_fee_rate = 1 - self._qkc_config.reward_tax_rate # type: Fraction
coinbase_tokens = {
self._qkc_config.genesis_token:
shard_config.COINBASE_AMOUNT * local_fee_rate.numerator //
local_fee_rate.denominator
}
coinbase_address = Address.create_empty_account(full_shard_id)
header = MinorBlockHeader(
version=genesis.VERSION,
height=genesis.HEIGHT,
branch=branch,
hash_prev_minor_block=bytes.fromhex(genesis.HASH_PREV_MINOR_BLOCK),
hash_prev_root_block=root_block.header.get_hash(),
evm_gas_limit=genesis.GAS_LIMIT,
hash_meta=sha3_256(meta.serialize()),
coinbase_amount_map=TokenBalanceMap(coinbase_tokens),
coinbase_address=coinbase_address,
create_time=genesis.TIMESTAMP,
difficulty=genesis.DIFFICULTY,
extra_data=bytes.fromhex(genesis.EXTRA_DATA),
)
return (
MinorBlock(header=header, meta=meta, tx_list=[]),
TokenBalanceMap(coinbase_tokens),
)
def test_shard_synchronizer_with_fork(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_key=0)
with ClusterContext(2, acc1) as clusters:
# shutdown cluster connection
clusters[1].peer.close()
block_list = []
# cluster 0 has 13 blocks added
shard_state0 = clusters[0].get_shard_state(0b10)
for i in range(13):
block = _tip_gen(shard_state0)
add_result = call_async(clusters[0].master.add_raw_minor_block(
block.header.branch, block.serialize()))
self.assertTrue(add_result)
block_list.append(block)
self.assertEqual(
clusters[0].get_shard_state(0b10).header_tip.height, 13)
# cluster 1 has 12 blocks added
shard_state0 = clusters[1].get_shard_state(0b10)
for i in range(12):
block = _tip_gen(shard_state0)
add_result = call_async(clusters[1].master.add_raw_minor_block(
block.header.branch, block.serialize()))
self.assertTrue(add_result)
self.assertEqual(
clusters[1].get_shard_state(0b10).header_tip.height, 12)
# reestablish cluster connection
call_async(clusters[1].network.connect(
"127.0.0.1",
clusters[0].master.env.cluster_config.SIMPLE_NETWORK.
BOOTSTRAP_PORT,
))
# a new block from cluster 0 will trigger sync in cluster 1
shard_state0 = clusters[0].get_shard_state(0b10)
block = _tip_gen(shard_state0)
add_result = call_async(clusters[0].master.add_raw_minor_block(
block.header.branch, block.serialize()))
self.assertTrue(add_result)
block_list.append(block)
# expect cluster 1 has all the blocks from cluter 0 and
# has the same tip as cluster 0
for block in block_list:
assert_true_with_timeout(
lambda: clusters[1].slave_list[0].shards[Branch(0b10)].
state.contain_block_by_hash(block.header.get_hash()))
assert_true_with_timeout(
lambda: clusters[1].master.root_state.db.
contain_minor_block_by_hash(block.header.get_hash()))
self.assertEqual(
clusters[1].get_shard_state(0b10).header_tip,
clusters[0].get_shard_state(0b10).header_tip,
)
async def __init_shard(shard):
await shard.init_from_root_block(root_block)
await shard.create_peer_shard_connections(self.cluster_peer_ids,
self.master)
branch = Branch(shard.full_shard_id)
self.shards[branch] = shard
if self.mining:
shard.miner.start()
def execute_tx(self, tx: TypedTransaction, from_address) -> Optional[bytes]:
evm_tx = tx.tx.to_evm_tx()
evm_tx.set_quark_chain_config(self.env.quark_chain_config)
branch = Branch(evm_tx.from_full_shard_id)
shard = self.shards.get(branch, None)
if not shard:
return None
return shard.state.execute_tx(tx, from_address)
async def submitWork(self, full_shard_key, header_hash, nonce, mixhash):
branch = None # `None` means getting work from root chain
if full_shard_key is not None:
branch = Branch(
self.master.env.quark_chain_config.
get_full_shard_id_by_full_shard_key(full_shard_key))
return await self.master.submit_work(branch, header_hash, nonce,
mixhash)
def add_tx(self, tx: Transaction) -> bool:
evm_tx = tx.code.get_evm_transaction()
evm_tx.set_quark_chain_config(self.env.quark_chain_config)
branch = Branch(evm_tx.from_full_shard_id)
shard = self.shards.get(branch, None)
if not shard:
return False
return shard.add_tx(tx)
def get_token_balance(self, address):
branch = Branch(
self.env.quark_chain_config.get_full_shard_id_by_full_shard_key(
address.full_shard_key))
shard = self.shards.get(branch, None)
if not shard:
return None
return shard.state.get_token_balance(address.recipient)
def estimate_gas(self, tx, from_address) -> Optional[int]:
evm_tx = tx.code.get_evm_transaction()
evm_tx.set_quark_chain_config(self.env.quark_chain_config)
branch = Branch(evm_tx.from_full_shard_id)
shard = self.shards.get(branch, None)
if not shard:
return None
return shard.state.estimate_gas(tx, from_address)
def get_transaction_list_by_address(self, address, start, limit):
branch = Branch(
self.env.quark_chain_config.get_full_shard_id_by_full_shard_key(
address.full_shard_key))
shard = self.shards.get(branch, None)
if not shard:
return None
return shard.state.get_transaction_list_by_address(
address, start, limit)
def get_code(self, address: Address,
block_height: Optional[int]) -> Optional[bytes]:
branch = Branch(
self.env.quark_chain_config.get_full_shard_id_by_full_shard_key(
address.full_shard_key))
shard = self.shards.get(branch, None)
if not shard:
return None
return shard.state.get_code(address.recipient, block_height)
def test_add_transaction(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_key=0)
acc2 = Address.create_from_identity(id1, full_shard_key=1)
with ClusterContext(2, acc1) as clusters:
master = clusters[0].master
slaves = clusters[0].slave_list
branch0 = Branch(2)
tx1 = create_transfer_transaction(
shard_state=slaves[0].shards[branch0].state,
key=id1.get_key(),
from_address=acc1,
to_address=acc1,
value=12345,
)
self.assertTrue(call_async(master.add_transaction(tx1)))
self.assertEqual(len(slaves[0].shards[branch0].state.tx_queue), 1)
branch1 = Branch(2 | 1)
tx2 = create_transfer_transaction(
shard_state=slaves[1].shards[branch1].state,
key=id1.get_key(),
from_address=acc2,
to_address=acc1,
value=12345,
gas=30000,
)
self.assertTrue(call_async(master.add_transaction(tx2)))
self.assertEqual(len(slaves[1].shards[branch1].state.tx_queue), 1)
# check the tx is received by the other cluster
tx_queue = clusters[1].slave_list[0].shards[branch0].state.tx_queue
assert_true_with_timeout(lambda: len(tx_queue) == 1)
self.assertEqual(tx_queue.pop_transaction(),
tx1.code.get_evm_transaction())
tx_queue = clusters[1].slave_list[1].shards[branch1].state.tx_queue
assert_true_with_timeout(lambda: len(tx_queue) == 1)
self.assertEqual(tx_queue.pop_transaction(),
tx2.code.get_evm_transaction())
def test_get_minor_block_by_hash(self):
db = ShardDbOperator(InMemoryDb(), DEFAULT_ENV, Branch(2))
block = MinorBlock(MinorBlockHeader(), MinorBlockMeta())
block_hash = block.header.get_hash()
db.put_minor_block(block, [])
self.assertEqual(db.get_minor_block_by_hash(block_hash), block)
self.assertIsNone(db.get_minor_block_by_hash(b""))
self.assertEqual(db.get_minor_block_header_by_hash(block_hash),
block.header)
self.assertIsNone(db.get_minor_block_header_by_hash(b""))
async def getTransactionReceipt(self, tx_id):
tx_hash, full_shard_key = tx_id
branch = Branch(
self.master.env.quark_chain_config.
get_full_shard_id_by_full_shard_key(full_shard_key))
resp = await self.master.get_transaction_receipt(tx_hash, branch)
if not resp:
return None
minor_block, i, receipt = resp
return receipt_encoder(minor_block, i, receipt)
async def gasPrice(self, full_shard_key: int):
full_shard_key = shard_id_decoder(full_shard_key)
if full_shard_key is None:
return None
branch = Branch(
self.master.env.quark_chain_config.
get_full_shard_id_by_full_shard_key(full_shard_key))
ret = await self.master.gas_price(branch)
if ret is None:
return None
return quantity_encoder(ret)
async def getTransactionById(self, tx_id):
tx_hash, full_shard_key = tx_id
branch = Branch(
self.master.env.quark_chain_config.
get_full_shard_id_by_full_shard_key(full_shard_key))
minor_block, i = await self.master.get_transaction_by_hash(
tx_hash, branch)
if not minor_block:
return None
if len(minor_block.tx_list) <= i:
return None
return tx_encoder(minor_block, i)
async def getMinorBlockById(self, block_id, include_transactions=False):
block_hash, full_shard_key = block_id
try:
branch = Branch(
self.master.env.quark_chain_config.
get_full_shard_id_by_full_shard_key(full_shard_key))
except Exception:
return None
block = await self.master.get_minor_block_by_hash(block_hash, branch)
if not block:
return None
return minor_block_encoder(block, include_transactions)
async def getWork(self, full_shard_key):
branch = None # `None` means getting work from root chain
if full_shard_key is not None:
branch = Branch(
self.master.env.quark_chain_config.
get_full_shard_id_by_full_shard_key(full_shard_key))
ret = await self.master.get_work(branch)
if ret is None:
return None
return [
data_encoder(ret.hash),
quantity_encoder(ret.height),
quantity_encoder(ret.difficulty),
]
async def gasPrice(self, full_shard_key: str, token_id: Optional[str] = None):
full_shard_key = shard_id_decoder(full_shard_key)
if full_shard_key is None:
return None
parsed_token_id = (
quantity_decoder(token_id) if token_id else token_id_encode("QKC")
)
branch = Branch(
self.master.env.quark_chain_config.get_full_shard_id_by_full_shard_key(
full_shard_key
)
)
ret = await self.master.gas_price(branch, parsed_token_id)
if ret is None:
return None
return quantity_encoder(ret)
async def getMinorBlockByHeight(self,
full_shard_key: int,
height=None,
include_transactions=False):
if height is not None:
height = quantity_decoder(height)
try:
branch = Branch(
self.master.env.quark_chain_config.
get_full_shard_id_by_full_shard_key(full_shard_key))
except Exception:
return None
block = await self.master.get_minor_block_by_height(height, branch)
if not block:
return None
return minor_block_encoder(block, include_transactions)
async def create_shards(self, root_block: RootBlock):
""" Create shards based on GENESIS config and root block height if they have
not been created yet."""
futures = []
for (full_shard_id, shard_config) in self.env.quark_chain_config.shards.items():
branch = Branch(full_shard_id)
if branch in self.shards:
continue
if not self.__cover_shard_id(full_shard_id) or not shard_config.GENESIS:
continue
if root_block.header.height >= shard_config.GENESIS.ROOT_HEIGHT:
shard = Shard(self.env, full_shard_id, self)
futures.append(shard.init_from_root_block(root_block))
self.shards[branch] = shard
if self.mining:
shard.miner.start()
await asyncio.gather(*futures)
def test_add_transaction(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_key=0)
acc2 = Address.create_from_identity(id1, full_shard_key=1)
with ClusterContext(2, acc1) as clusters:
master = clusters[0].master
root = call_async(
master.get_next_block_to_mine(acc1, branch_value=None))
call_async(master.add_root_block(root))
tx1 = create_transfer_transaction(
shard_state=clusters[0].get_shard_state(0b10),
key=id1.get_key(),
from_address=acc1,
to_address=acc1,
value=12345,
)
self.assertTrue(call_async(master.add_transaction(tx1)))
self.assertEqual(len(clusters[0].get_shard_state(0b10).tx_queue),
1)
branch1 = Branch(2 | 1)
tx2 = create_transfer_transaction(
shard_state=clusters[0].get_shard_state(0b11),
key=id1.get_key(),
from_address=acc2,
to_address=acc1,
value=12345,
gas=30000,
)
self.assertTrue(call_async(master.add_transaction(tx2)))
self.assertEqual(len(clusters[0].get_shard_state(0b11).tx_queue),
1)
# check the tx is received by the other cluster
tx_queue = clusters[1].get_shard_state(0b10).tx_queue
assert_true_with_timeout(lambda: len(tx_queue) == 1)
self.assertEqual(tx_queue.pop_transaction(), tx1.tx.to_evm_tx())
tx_queue = clusters[1].get_shard_state(0b11).tx_queue
assert_true_with_timeout(lambda: len(tx_queue) == 1)
self.assertEqual(tx_queue.pop_transaction(), tx2.tx.to_evm_tx())
def test_get_primary_account_data(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_key=0)
acc2 = Address.create_random_account(full_shard_key=1)
with ClusterContext(1, acc1) as clusters:
master = clusters[0].master
slaves = clusters[0].slave_list
branch = Branch(2)
self.assertEqual(
call_async(
master.get_primary_account_data(acc1)).transaction_count,
0)
tx = create_transfer_transaction(
shard_state=slaves[0].shards[branch].state,
key=id1.get_key(),
from_address=acc1,
to_address=acc1,
value=12345,
)
self.assertTrue(slaves[0].add_tx(tx))
is_root, root = call_async(
master.get_next_block_to_mine(address=acc1, prefer_root=True))
self.assertTrue(is_root)
call_async(master.add_root_block(root))
is_root, block1 = call_async(
master.get_next_block_to_mine(address=acc1))
self.assertFalse(is_root)
self.assertTrue(
call_async(
master.add_raw_minor_block(block1.header.branch,
block1.serialize())))
self.assertEqual(
call_async(
master.get_primary_account_data(acc1)).transaction_count,
1)
self.assertEqual(
call_async(
master.get_primary_account_data(acc2)).transaction_count,
0)
