async def eth_getBlockByNumber(self, block_height, include_transactions):
"""
NOTE: only support block_id "latest" or hex
"""
def block_transcoder(block):
"""
QuarkChain Block => ETH Block
"""
return {
**block,
"number": block["height"],
"parentHash": block["hashPrevMinorBlock"],
"sha3Uncles": "",
"logsBloom": "",
"transactionsRoot": block["hashMerkleRoot"], # ?
"stateRoot": block["hashEvmStateRoot"], # ?
}
block = await self.master.get_minor_block_by_height(
block_height, Branch.create(self.master.get_shard_size(), 0)
)
if block is None:
return None
return block_transcoder(minor_block_encoder(block))
def test_get_primary_account_data(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_id=0)
acc2 = Address.create_random_account(full_shard_id=1)
with ClusterContext(1, acc1) as clusters:
master = clusters[0].master
slaves = clusters[0].slave_list
branch = Branch.create(2, 0)
self.assertEqual(
call_async(master.get_primary_account_data(acc1)).transaction_count, 0
)
tx = create_transfer_transaction(
shard_state=slaves[0].shards[branch].state,
key=id1.get_key(),
from_address=acc1,
to_address=acc1,
value=12345,
)
self.assertTrue(slaves[0].add_tx(tx))
is_root, block1 = call_async(master.get_next_block_to_mine(address=acc1))
self.assertTrue(
call_async(
master.add_raw_minor_block(block1.header.branch, block1.serialize())
)
)
self.assertEqual(
call_async(master.get_primary_account_data(acc1)).transaction_count, 1
)
self.assertEqual(
call_async(master.get_primary_account_data(acc2)).transaction_count, 0
)
def get_shard(self, shard_id) -> Shard:
branch = Branch.create(self.master.env.quark_chain_config.SHARD_SIZE,
shard_id)
for slave in self.slave_list:
if branch in slave.shards:
return slave.shards[branch]
return None
def get_balance(self, address):
branch = Branch.create(self.__get_shard_size(),
address.get_shard_id(self.__get_shard_size()))
shard = self.shards.get(branch, None)
if not shard:
return None
return shard.state.get_balance(address.recipient)
async def getTransactionConfirmedByNumberRootBlocks(self, tx_id):
tx_hash, full_shard_key = tx_id
branch = Branch(
self.master.env.quark_chain_config.get_full_shard_id_by_full_shard_key(
full_shard_key
)
)
minor_block, i = await self.master.get_transaction_by_hash(tx_hash, branch)
if not minor_block:
return None
if len(minor_block.tx_list) <= i:
return None
confirming_hash = self.master.root_state.db.get_root_block_confirming_minor_block(
minor_block.header.get_hash()
+ minor_block.header.branch.get_full_shard_id().to_bytes(4, byteorder="big")
)
if confirming_hash is None:
return quantity_encoder(0)
confirming_header = self.master.root_state.db.get_root_block_header_by_hash(
confirming_hash
)
canonical_hash = self.master.root_state.db.get_root_block_hash_by_height(
confirming_header.height
)
if canonical_hash != confirming_hash:
return quantity_encoder(0)
tip = self.master.root_state.tip
return quantity_encoder(tip.height - confirming_header.height + 1)
def test_gasPrice(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_id=0)
with ClusterContext(
1, acc1, small_coinbase=True) as clusters, jrpc_server_context(
clusters[0].master):
master = clusters[0].master
slaves = clusters[0].slave_list
branch = Branch.create(2, 0)
# run for multiple times
for _ in range(3):
tx = create_transfer_transaction(
shard_state=slaves[0].shards[branch].state,
key=id1.get_key(),
from_address=acc1,
to_address=acc1,
value=0,
gas_price=12,
)
self.assertTrue(slaves[0].add_tx(tx))
_, block = call_async(
master.get_next_block_to_mine(address=acc1))
self.assertTrue(
call_async(clusters[0].get_shard(0).add_block(block)))
for using_eth_endpoint in (True, False):
if using_eth_endpoint:
resp = send_request("eth_gasPrice", "0x0")
else:
resp = send_request("gasPrice", "0x0")
self.assertEqual(resp, "0xc")
def create_minor_block(self, root_block: RootBlock, full_shard_id: int,
evm_state: EvmState) -> MinorBlock:
""" Create genesis block for shard.
Genesis block's hash_prev_root_block is set to the genesis root block.
Genesis state will be committed to the given evm_state.
Based on ALLOC, genesis_token will be added to initial accounts.
"""
branch = Branch(full_shard_id)
shard_config = self._qkc_config.shards[full_shard_id]
genesis = shard_config.GENESIS
for address_hex, alloc_amount in genesis.ALLOC.items():
address = Address.create_from(bytes.fromhex(address_hex))
check(
self._qkc_config.get_full_shard_id_by_full_shard_key(
address.full_shard_key) == full_shard_id)
evm_state.full_shard_key = address.full_shard_key
if isinstance(alloc_amount, dict):
for k, v in alloc_amount.items():
evm_state.delta_token_balance(address.recipient,
token_id_encode(k), v)
else:
evm_state.delta_token_balance(address.recipient,
self._qkc_config.genesis_token,
alloc_amount)
evm_state.commit()
meta = MinorBlockMeta(
hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT),
hash_evm_state_root=evm_state.trie.root_hash,
)
local_fee_rate = 1 - self._qkc_config.reward_tax_rate # type: Fraction
coinbase_tokens = {
self._qkc_config.genesis_token:
shard_config.COINBASE_AMOUNT * local_fee_rate.numerator //
local_fee_rate.denominator
}
coinbase_address = Address.create_empty_account(full_shard_id)
header = MinorBlockHeader(
version=genesis.VERSION,
height=genesis.HEIGHT,
branch=branch,
hash_prev_minor_block=bytes.fromhex(genesis.HASH_PREV_MINOR_BLOCK),
hash_prev_root_block=root_block.header.get_hash(),
evm_gas_limit=genesis.GAS_LIMIT,
hash_meta=sha3_256(meta.serialize()),
coinbase_amount_map=TokenBalanceMap(coinbase_tokens),
coinbase_address=coinbase_address,
create_time=genesis.TIMESTAMP,
difficulty=genesis.DIFFICULTY,
extra_data=bytes.fromhex(genesis.EXTRA_DATA),
)
return (
MinorBlock(header=header, meta=meta, tx_list=[]),
TokenBalanceMap(coinbase_tokens),
)
def test_sendTransaction_with_bad_signature(self):
""" sendTransaction validates signature """
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_id=0)
acc2 = Address.create_random_account(full_shard_id=1)
with ClusterContext(
1, acc1, small_coinbase=True) as clusters, jrpc_server_context(
clusters[0].master):
slaves = clusters[0].slave_list
branch = Branch.create(2, 0)
request = dict(
to="0x" + acc2.recipient.hex(),
gasPrice="0x6",
gas=hex(30000),
value="0xf",
v="0x1",
r="0x2",
s="0x3",
nonce="0x0",
fromFullShardId="0x00000000",
toFullShardId="0x00000001",
)
self.assertIsNone(send_request("sendTransaction", [request]))
self.assertEqual(len(slaves[0].shards[branch].state.tx_queue), 0)
def test_getTransactionById(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_id=0)
with ClusterContext(
1, acc1, small_coinbase=True) as clusters, jrpc_server_context(
clusters[0].master):
master = clusters[0].master
slaves = clusters[0].slave_list
branch = Branch.create(2, 0)
self.assertEqual(
call_async(
master.get_primary_account_data(acc1)).transaction_count,
0)
tx = create_transfer_transaction(
shard_state=slaves[0].shards[branch].state,
key=id1.get_key(),
from_address=acc1,
to_address=acc1,
value=12345,
)
self.assertTrue(slaves[0].add_tx(tx))
_, block1 = call_async(master.get_next_block_to_mine(address=acc1))
self.assertTrue(
call_async(clusters[0].get_shard(0).add_block(block1)))
resp = send_request(
"getTransactionById",
"0x" + tx.get_hash().hex() +
acc1.full_shard_id.to_bytes(4, "big").hex(),
)
self.assertEqual(resp["hash"], "0x" + tx.get_hash().hex())
def test_getTransactionReceipt_on_transfer(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_id=0)
with ClusterContext(1, acc1) as clusters, jrpc_server_context(
clusters[0].master):
master = clusters[0].master
slaves = clusters[0].slave_list
branch = Branch.create(2, 0)
tx = create_transfer_transaction(
shard_state=slaves[0].shards[branch].state,
key=id1.get_key(),
from_address=acc1,
to_address=acc1,
value=12345,
)
self.assertTrue(slaves[0].add_tx(tx))
_, block1 = call_async(master.get_next_block_to_mine(address=acc1))
self.assertTrue(
call_async(clusters[0].get_shard(0).add_block(block1)))
for endpoint in ("getTransactionReceipt",
"eth_getTransactionReceipt"):
resp = send_request(
endpoint,
"0x" + tx.get_hash().hex() +
acc1.full_shard_id.to_bytes(4, "big").hex(),
)
self.assertEqual(resp["transactionHash"],
"0x" + tx.get_hash().hex())
self.assertEqual(resp["status"], "0x1")
self.assertEqual(resp["cumulativeGasUsed"], "0x5208")
self.assertIsNone(resp["contractAddress"])
async def create_shards(self, root_block: RootBlock):
""" Create shards based on GENESIS config and root block height if they have
not been created yet."""
async def __init_shard(shard):
await shard.init_from_root_block(root_block)
await shard.create_peer_shard_connections(self.cluster_peer_ids,
self.master)
branch = Branch(shard.full_shard_id)
self.shards[branch] = shard
if self.mining:
shard.miner.start()
new_shards = []
for (full_shard_id,
shard_config) in self.env.quark_chain_config.shards.items():
branch = Branch(full_shard_id)
if branch in self.shards:
continue
if not self.__cover_shard_id(
full_shard_id) or not shard_config.GENESIS:
continue
if root_block.header.height >= shard_config.GENESIS.ROOT_HEIGHT:
new_shards.append(Shard(self.env, full_shard_id, self))
await asyncio.gather(*[__init_shard(shard) for shard in new_shards])
def add_peer(self, peer: PeerShardConnection):
self.peers[peer.cluster_peer_id] = peer
Logger.info(
"[{}] connected to peer {}".format(
Branch(self.full_shard_id).to_str(), peer.cluster_peer_id
)
)
async def _get_logs(self, data, shard, decoder: Callable[[str], bytes]):
start_block = data.get("fromBlock", "latest")
end_block = data.get("toBlock", "latest")
# TODO: not supported yet for "earliest" or "pending" block
if (isinstance(start_block, str) and start_block != "latest") or (
isinstance(end_block, str) and end_block != "latest"
):
return None
# parse addresses / topics
addresses, topics = [], []
if "address" in data:
if isinstance(data["address"], str):
addresses = [Address.deserialize(decoder(data["address"]))]
elif isinstance(data["address"], list):
addresses = [Address.deserialize(decoder(a)) for a in data["address"]]
if shard is not None:
addresses = [Address(a.recipient, shard) for a in addresses]
if "topics" in data:
for topic_item in data["topics"]:
if isinstance(topic_item, str):
topics.append([data_decoder(topic_item)])
elif isinstance(topic_item, list):
topics.append([data_decoder(tp) for tp in topic_item])
branch = Branch.create(self.master.get_shard_size(), shard)
logs = await self.master.get_logs(
addresses, topics, start_block, end_block, branch
)
if logs is None:
return None
return loglist_encoder(logs)
async def getTransactionConfirmedByNumberRootBlocks(self, tx_id):
tx_hash, full_shard_key = tx_id
branch = Branch(
self.master.env.quark_chain_config.
get_full_shard_id_by_full_shard_key(full_shard_key))
minor_block, i = await self.master.get_transaction_by_hash(
tx_hash, branch)
if not minor_block:
return None
if len(minor_block.tx_list) <= i:
return None
root_hash = self.master.root_state.db.get_root_block_confirming_minor_block(
minor_block.header.get_hash() +
minor_block.header.branch.get_full_shard_id().to_bytes(
4, byteorder="big"))
if root_hash is None:
return quantity_encoder(0)
root_header_tip = self.master.root_state.tip
root_header = self.master.root_state.db.get_root_block_header_by_hash(
root_hash, consistency_check=False)
if not self.master.root_state.is_same_chain(root_header_tip,
root_header):
return quantity_encoder(0)
return quantity_encoder(root_header_tip.height - root_header.height +
1)
def is_neighbor(b1: Branch, b2: Branch):
"""A naive algorithm to decide neighbor relationship
TODO: a better algorithm, because the current one ensures 32 neighbors ONLY when there are 2^32 shards
"""
if b1.get_chain_id() == b2.get_chain_id():
return is_p2(abs(b1.get_shard_id() - b2.get_shard_id()))
if b1.get_shard_id() == b2.get_shard_id():
return is_p2(abs(b1.get_chain_id() - b2.get_chain_id()))
return False
def test_shard_synchronizer_with_fork(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_key=0)
with ClusterContext(2, acc1) as clusters:
# shutdown cluster connection
clusters[1].peer.close()
block_list = []
# cluster 0 has 13 blocks added
shard_state0 = clusters[0].get_shard_state(0b10)
for i in range(13):
block = _tip_gen(shard_state0)
add_result = call_async(clusters[0].master.add_raw_minor_block(
block.header.branch, block.serialize()))
self.assertTrue(add_result)
block_list.append(block)
self.assertEqual(
clusters[0].get_shard_state(0b10).header_tip.height, 13)
# cluster 1 has 12 blocks added
shard_state0 = clusters[1].get_shard_state(0b10)
for i in range(12):
block = _tip_gen(shard_state0)
add_result = call_async(clusters[1].master.add_raw_minor_block(
block.header.branch, block.serialize()))
self.assertTrue(add_result)
self.assertEqual(
clusters[1].get_shard_state(0b10).header_tip.height, 12)
# reestablish cluster connection
call_async(clusters[1].network.connect(
"127.0.0.1",
clusters[0].master.env.cluster_config.SIMPLE_NETWORK.
BOOTSTRAP_PORT,
))
# a new block from cluster 0 will trigger sync in cluster 1
shard_state0 = clusters[0].get_shard_state(0b10)
block = _tip_gen(shard_state0)
add_result = call_async(clusters[0].master.add_raw_minor_block(
block.header.branch, block.serialize()))
self.assertTrue(add_result)
block_list.append(block)
# expect cluster 1 has all the blocks from cluter 0 and
# has the same tip as cluster 0
for block in block_list:
assert_true_with_timeout(
lambda: clusters[1].slave_list[0].shards[Branch(0b10)].
state.contain_block_by_hash(block.header.get_hash()))
assert_true_with_timeout(
lambda: clusters[1].master.root_state.db.
contain_minor_block_by_hash(block.header.get_hash()))
self.assertEqual(
clusters[1].get_shard_state(0b10).header_tip,
clusters[0].get_shard_state(0b10).header_tip,
)
def test_getMinorBlock(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_id=0)
with ClusterContext(
1, acc1, small_coinbase=True) as clusters, jrpc_server_context(
clusters[0].master):
master = clusters[0].master
slaves = clusters[0].slave_list
branch = Branch.create(2, 0)
self.assertEqual(
call_async(
master.get_primary_account_data(acc1)).transaction_count,
0)
tx = create_transfer_transaction(
shard_state=slaves[0].shards[branch].state,
key=id1.get_key(),
from_address=acc1,
to_address=acc1,
value=12345,
)
self.assertTrue(slaves[0].add_tx(tx))
_, block1 = call_async(master.get_next_block_to_mine(address=acc1))
self.assertTrue(
call_async(clusters[0].get_shard(0).add_block(block1)))
# By id
resp = send_request(
"getMinorBlockById",
"0x" + block1.header.get_hash().hex() + "0" * 8,
False,
)
self.assertEqual(resp["transactions"][0],
"0x" + tx.get_hash().hex() + "0" * 8)
resp = send_request(
"getMinorBlockById",
"0x" + block1.header.get_hash().hex() + "0" * 8,
True,
)
self.assertEqual(resp["transactions"][0]["hash"],
"0x" + tx.get_hash().hex())
resp = send_request("getMinorBlockById", "0x" + "ff" * 36, True)
self.assertIsNone(resp)
# By height
resp = send_request("getMinorBlockByHeight", "0x0", "0x1", False)
self.assertEqual(resp["transactions"][0],
"0x" + tx.get_hash().hex() + "0" * 8)
resp = send_request("getMinorBlockByHeight", "0x0", "0x1", True)
self.assertEqual(resp["transactions"][0]["hash"],
"0x" + tx.get_hash().hex())
resp = send_request("getMinorBlockByHeight", "0x1", "0x2", False)
self.assertIsNone(resp)
resp = send_request("getMinorBlockByHeight", "0x0", "0x4", False)
self.assertIsNone(resp)
def estimate_gas(self, tx, from_address) -> Optional[int]:
evm_tx = tx.code.get_evm_transaction()
evm_tx.set_quark_chain_config(self.env.quark_chain_config)
branch = Branch(evm_tx.from_full_shard_id)
shard = self.shards.get(branch, None)
if not shard:
return None
return shard.state.estimate_gas(tx, from_address)
def get_token_balance(self, address):
branch = Branch(
self.env.quark_chain_config.get_full_shard_id_by_full_shard_key(
address.full_shard_key))
shard = self.shards.get(branch, None)
if not shard:
return None
return shard.state.get_token_balance(address.recipient)
def add_tx(self, tx: Transaction) -> bool:
evm_tx = tx.code.get_evm_transaction()
evm_tx.set_quark_chain_config(self.env.quark_chain_config)
branch = Branch(evm_tx.from_full_shard_id)
shard = self.shards.get(branch, None)
if not shard:
return False
return shard.add_tx(tx)
async def submitWork(self, full_shard_key, header_hash, nonce, mixhash):
branch = None # `None` means getting work from root chain
if full_shard_key is not None:
branch = Branch(
self.master.env.quark_chain_config.
get_full_shard_id_by_full_shard_key(full_shard_key))
return await self.master.submit_work(branch, header_hash, nonce,
mixhash)
async def __init_shard(shard):
await shard.init_from_root_block(root_block)
await shard.create_peer_shard_connections(self.cluster_peer_ids,
self.master)
branch = Branch(shard.full_shard_id)
self.shards[branch] = shard
if self.mining:
shard.miner.start()
async def getMinorBlockById(self, block_id, include_transactions=False):
block_hash, full_shard_id = block_id
shard_size = self.master.get_shard_size()
branch = Branch.create(shard_size, (shard_size - 1) & full_shard_id)
block = await self.master.get_minor_block_by_hash(block_hash, branch)
if not block:
return None
return minor_block_encoder(block, include_transactions)
def add_tx(self, tx: Transaction) -> bool:
evm_tx = tx.code.get_evm_transaction()
evm_tx.set_shard_size(self.__get_shard_size())
branch = Branch.create(self.__get_shard_size(), evm_tx.from_shard_id())
shard = self.shards.get(branch, None)
if not shard:
return False
return shard.add_tx(tx)
def execute_tx(self, tx: TypedTransaction, from_address) -> Optional[bytes]:
evm_tx = tx.tx.to_evm_tx()
evm_tx.set_quark_chain_config(self.env.quark_chain_config)
branch = Branch(evm_tx.from_full_shard_id)
shard = self.shards.get(branch, None)
if not shard:
return None
return shard.state.execute_tx(tx, from_address)
def estimate_gas(self, tx, from_address) -> Optional[int]:
evm_tx = tx.code.get_evm_transaction()
evm_tx.set_shard_size(self.__get_shard_size())
branch = Branch.create(self.__get_shard_size(), evm_tx.from_shard_id())
shard = self.shards.get(branch, None)
if not shard:
return None
return shard.state.estimate_gas(tx, from_address)
def get_transaction_list_by_address(self, address, start, limit):
branch = Branch.create(self.__get_shard_size(),
address.get_shard_id(self.__get_shard_size()))
shard = self.shards.get(branch, None)
if not shard:
return None
return shard.state.get_transaction_list_by_address(
address, start, limit)
def test_getStorageAt(self):
key = bytes.fromhex(
"c987d4506fb6824639f9a9e3b8834584f5165e94680501d1b0044071cd36c3b3")
id1 = Identity.create_from_key(key)
acc1 = Address.create_from_identity(id1, full_shard_id=0)
created_addr = "0x8531eb33bba796115f56ffa1b7df1ea3acdd8cdd00000000"
with ClusterContext(
1, acc1, small_coinbase=True) as clusters, jrpc_server_context(
clusters[0].master):
master = clusters[0].master
slaves = clusters[0].slave_list
branch = Branch.create(2, 0)
tx = create_contract_with_storage_transaction(
shard_state=slaves[0].shards[branch].state,
key=id1.get_key(),
from_address=acc1,
to_full_shard_id=acc1.full_shard_id,
)
self.assertTrue(slaves[0].add_tx(tx))
_, block = call_async(master.get_next_block_to_mine(address=acc1))
self.assertTrue(
call_async(clusters[0].get_shard(0).add_block(block)))
for using_eth_endpoint in (True, False):
if using_eth_endpoint:
req = lambda k: send_request("eth_getStorageAt",
created_addr[:-8], k, "0x0")
else:
req = lambda k: send_request("getStorageAt", created_addr,
k)
# first storage
response = req("0x0")
# equals 1234
self.assertEqual(
response,
"0x00000000000000000000000000000000000000000000000000000000000004d2",
)
# mapping storage
k = sha3_256(
bytes.fromhex(acc1.recipient.hex().zfill(64) +
"1".zfill(64)))
response = req("0x" + k.hex())
self.assertEqual(
response,
"0x000000000000000000000000000000000000000000000000000000000000162e",
)
# doesn't exist
response = req("0x3")
self.assertEqual(
response,
"0x0000000000000000000000000000000000000000000000000000000000000000",
)
def get_code(self, address: Address,
block_height: Optional[int]) -> Optional[bytes]:
shard_size = self.__get_shard_size()
shard_id = address.get_shard_id(shard_size)
branch = Branch.create(shard_size, shard_id)
shard = self.shards.get(branch, None)
if not shard:
return None
return shard.state.get_code(address.recipient, block_height)
async def gasPrice(self, shard):
shard = shard_id_decoder(shard)
if shard is None:
return None
branch = Branch.create(self.master.get_shard_size(), shard)
ret = await self.master.gas_price(branch)
if ret is None:
return None
return quantity_encoder(ret)
