async def __run_sync(self):
if self.__has_block_hash(self.header.get_hash()):
return
# descending height
block_header_chain = [self.header]
# TODO: Stop if too many headers to revert
while not self.__has_block_hash(
block_header_chain[-1].hash_prev_minor_block):
block_hash = block_header_chain[-1].hash_prev_minor_block
height = block_header_chain[-1].height - 1
if self.shard_state.header_tip.height - height > self.max_staleness:
Logger.warning(
"[{}] abort syncing due to forking at very old block {} << {}"
.format(
self.header.branch.get_shard_id(),
height,
self.shard_state.header_tip.height,
))
return
if not self.shard_state.db.contain_root_block_by_hash(
block_header_chain[-1].hash_prev_root_block):
return
Logger.info("[{}] downloading headers from {} {}".format(
self.shard_state.branch.get_shard_id(), height,
block_hash.hex()))
block_header_list = await asyncio.wait_for(
self.__download_block_headers(block_hash), TIMEOUT)
Logger.info("[{}] downloaded {} headers from peer".format(
self.shard_state.branch.get_shard_id(),
len(block_header_list)))
if not self.__validate_block_headers(block_header_list):
# TODO: tag bad peer
return self.shard_conn.close_with_error(
"Bad peer sending discontinuing block headers")
for header in block_header_list:
if self.__has_block_hash(header.get_hash()):
break
block_header_chain.append(header)
# ascending height
block_header_chain.reverse()
while len(block_header_chain) > 0:
block_chain = await asyncio.wait_for(
self.__download_blocks(block_header_chain[:100]), TIMEOUT)
Logger.info("[{}] downloaded {} blocks from peer".format(
self.shard_state.branch.get_shard_id(), len(block_chain)))
check(len(block_chain) == len(block_header_chain[:100]))
for block in block_chain:
# Stop if the block depends on an unknown root block
# TODO: move this check to early stage to avoid downloading unnecessary headers
if not self.shard_state.db.contain_root_block_by_hash(
block.header.hash_prev_root_block):
return
await self.shard.add_block(block)
block_header_chain.pop(0)
def mine_next_root_block(self, ts):
check(self.root_block_to_mine is None)
if (len(self.pending_minor_block_queue) == 0
or self.pending_minor_block_queue[0].root_block.block_height +
self.args.rprevious >
self.get_root_block_tip().block_height + 1):
# The root block is not able to mine. Will wait until a minor block is produced.
# TODO: the root block may also mine an null minor block with reduced coinbase reward
return
self.root_block_to_mine = RootBlock(
block_height=self.get_root_block_tip().block_height + 1,
minor_block_list=[], # to be fill once mined
mining_time=ts,
mined_time=None,
)
self.scheduler.schedule_after(
get_next_interval(ROOT_BLOCK_INTERVAL_SEC),
self.mine_root_block,
self.root_block_to_mine,
)
if self.args.verbose >= 1:
print("%0.2f: root_block %d mining ..." %
(ts, self.root_block_to_mine.block_height))
def get_test_env(
genesis_account=Address.create_empty_account(),
genesis_quarkash=0,
genesis_minor_quarkash=0,
shard_size=2,
genesis_root_heights=None,
):
env = DEFAULT_ENV.copy()
env.db = InMemoryDb()
env.set_network_id(1234567890)
env.cluster_config = ClusterConfig()
env.quark_chain_config.update(shard_size, 1, 1)
env.quark_chain_config.TESTNET_MASTER_ADDRESS = genesis_account.serialize(
).hex()
if genesis_root_heights:
check(len(genesis_root_heights) == shard_size)
for shard_id in range(shard_size):
env.quark_chain_config.SHARD_LIST[
shard_id].GENESIS.ROOT_HEIGHT = genesis_root_heights[shard_id]
# fund genesis account in all shards
for i, shard in enumerate(env.quark_chain_config.SHARD_LIST):
shard.GENESIS.ALLOC[genesis_account.address_in_shard(
i).serialize().hex()] = genesis_minor_quarkash
env.quark_chain_config.SKIP_MINOR_DIFFICULTY_CHECK = True
env.quark_chain_config.SKIP_ROOT_DIFFICULTY_CHECK = True
env.cluster_config.ENABLE_TRANSACTION_HISTORY = True
env.cluster_config.DB_PATH_ROOT = ""
check(env.cluster_config.use_mem_db())
return env
def create_minor_block(self, root_block: RootBlock, full_shard_id: int,
evm_state: EvmState) -> MinorBlock:
""" Create genesis block for shard.
Genesis block's hash_prev_root_block is set to the genesis root block.
Genesis state will be committed to the given evm_state.
Based on ALLOC, genesis_token will be added to initial accounts.
"""
branch = Branch(full_shard_id)
shard_config = self._qkc_config.shards[full_shard_id]
genesis = shard_config.GENESIS
for address_hex, alloc_amount in genesis.ALLOC.items():
address = Address.create_from(bytes.fromhex(address_hex))
check(
self._qkc_config.get_full_shard_id_by_full_shard_key(
address.full_shard_key) == full_shard_id)
evm_state.full_shard_key = address.full_shard_key
if isinstance(alloc_amount, dict):
for k, v in alloc_amount.items():
evm_state.delta_token_balance(address.recipient,
token_id_encode(k), v)
else:
evm_state.delta_token_balance(address.recipient,
self._qkc_config.genesis_token,
alloc_amount)
evm_state.commit()
meta = MinorBlockMeta(
hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT),
hash_evm_state_root=evm_state.trie.root_hash,
)
local_fee_rate = 1 - self._qkc_config.reward_tax_rate # type: Fraction
coinbase_tokens = {
self._qkc_config.genesis_token:
shard_config.COINBASE_AMOUNT * local_fee_rate.numerator //
local_fee_rate.denominator
}
coinbase_address = Address.create_empty_account(full_shard_id)
header = MinorBlockHeader(
version=genesis.VERSION,
height=genesis.HEIGHT,
branch=branch,
hash_prev_minor_block=bytes.fromhex(genesis.HASH_PREV_MINOR_BLOCK),
hash_prev_root_block=root_block.header.get_hash(),
evm_gas_limit=genesis.GAS_LIMIT,
hash_meta=sha3_256(meta.serialize()),
coinbase_amount_map=TokenBalanceMap(coinbase_tokens),
coinbase_address=coinbase_address,
create_time=genesis.TIMESTAMP,
difficulty=genesis.DIFFICULTY,
extra_data=bytes.fromhex(genesis.EXTRA_DATA),
)
return (
MinorBlock(header=header, meta=meta, tx_list=[]),
TokenBalanceMap(coinbase_tokens),
)
def __get_branch_to_add_xshard_tx_list_request(self, block_hash,
xshard_tx_list,
prev_root_height):
xshard_map = dict(
) # type: Dict[Branch, List[CrossShardTransactionDeposit]]
# only broadcast to the shards that have been initialized
initialized_shard_ids = self.env.quark_chain_config.get_initialized_shard_ids_before_root_height(
prev_root_height)
for shard_id in initialized_shard_ids:
branch = Branch.create(self.__get_shard_size(), shard_id)
xshard_map[branch] = []
for xshard_tx in xshard_tx_list:
shard_id = xshard_tx.to_address.get_shard_id(
self.__get_shard_size())
branch = Branch.create(self.__get_shard_size(), shard_id)
check(branch in xshard_map)
xshard_map[branch].append(xshard_tx)
branch_to_add_xshard_tx_list_request = (
dict()) # type: Dict[Branch, AddXshardTxListRequest]
for branch, tx_list in xshard_map.items():
cross_shard_tx_list = CrossShardTransactionList(tx_list)
request = AddXshardTxListRequest(branch, block_hash,
cross_shard_tx_list)
branch_to_add_xshard_tx_list_request[branch] = request
return branch_to_add_xshard_tx_list_request
def put_root_block_index(self, block):
self.db.put(b"ri_%d" % block.header.height, block.header.get_hash())
if not self.count_minor_blocks:
return
# Count minor blocks by miner address
shard_size = block.header.shard_info.get_shard_size()
if block.header.height > 0:
shard_recipient_cnt = self.get_block_count(block.header.height - 1,
shard_size)
else:
shard_recipient_cnt = [dict() for _ in range(shard_size)]
for header in block.minor_block_header_list:
shard = header.branch.get_shard_id()
recipient = header.coinbase_address.recipient.hex()
shard_recipient_cnt[shard][recipient] = (
shard_recipient_cnt[shard].get(recipient, 0) + 1)
for shard, r_c in enumerate(shard_recipient_cnt):
data = bytearray()
for recipient, count in r_c.items():
data.extend(bytes.fromhex(recipient))
data.extend(count.to_bytes(4, "big"))
check(len(data) % 24 == 0)
self.db.put(b"count_%d_%d" % (shard, block.header.height), data)
async def broadcast_xshard_tx_list(self, block, xshard_tx_list,
prev_root_height):
""" Broadcast x-shard transactions to their recipient shards """
block_hash = block.header.get_hash()
branch_to_add_xshard_tx_list_request = self.__get_branch_to_add_xshard_tx_list_request(
block_hash, xshard_tx_list, prev_root_height)
rpc_futures = []
for branch, request in branch_to_add_xshard_tx_list_request.items():
if branch == block.header.branch or not is_neighbor(
block.header.branch, branch):
continue
if branch in self.shards:
self.shards[
branch].state.add_cross_shard_tx_list_by_minor_block_hash(
block_hash, request.tx_list)
for slave_conn in self.slave_connection_manager.get_connections_by_shard(
branch.get_shard_id()):
future = slave_conn.write_rpc_request(
ClusterOp.ADD_XSHARD_TX_LIST_REQUEST, request)
rpc_futures.append(future)
responses = await asyncio.gather(*rpc_futures)
check(all([response.error_code == 0 for _, response, _ in responses]))
def __init__(
self,
num_cluster,
genesis_account=Address.create_empty_account(),
chain_size=2,
shard_size=2,
num_slaves=None,
genesis_root_heights=None,
remote_mining=False,
small_coinbase=False,
loadtest_accounts=None,
connect=True,
should_set_gas_price_limit=False,
mblock_coinbase_amount=None,
genesis_minor_quarkash=1000000,
):
self.num_cluster = num_cluster
self.genesis_account = genesis_account
self.chain_size = chain_size
self.shard_size = shard_size
self.num_slaves = num_slaves if num_slaves else chain_size
self.genesis_root_heights = genesis_root_heights
self.remote_mining = remote_mining
self.small_coinbase = small_coinbase
self.loadtest_accounts = loadtest_accounts
self.connect = connect
self.should_set_gas_price_limit = should_set_gas_price_limit
self.mblock_coinbase_amount = mblock_coinbase_amount
self.genesis_minor_quarkash = genesis_minor_quarkash
check(is_p2(self.num_slaves))
check(is_p2(self.shard_size))
def get_next_minor_block_to_mine(self, ts):
tip = self.get_minor_block_tip()
# Produce block in this epoch if the epoch has space
if tip.root_index < MINOR_BLOCK_PER_ROOT_BLOCK - 1:
check(ts == tip.mined_time)
return MinorBlock(
root_block=tip.root_block,
root_index=tip.root_index + 1,
block_height=tip.block_height + 1,
mined_time=ts + MINOR_BLOCK_INTERVAL_SEC,
)
# Produce block in next epoch if a root block is available
root_height_to_confirm = tip.root_block.block_height + 1
if root_height_to_confirm <= self.get_root_block_tip().block_height:
root_block = self.root_block_list[root_height_to_confirm]
check(ts == max(tip.mined_time, root_block.mined_time))
return MinorBlock(
root_block=root_block,
root_index=0,
block_height=tip.block_height + 1,
mined_time=ts + MINOR_BLOCK_INTERVAL_SEC,
)
# Unable to find a root block to produce the minor block
return None
def __init__(self, env, diff_calc=None):
self.env = env
self.root_config = env.quark_chain_config.ROOT
if not diff_calc:
cutoff = self.root_config.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME
diff_factor = self.root_config.DIFFICULTY_ADJUSTMENT_FACTOR
min_diff = self.root_config.GENESIS.DIFFICULTY
check(cutoff > 0 and diff_factor > 0 and min_diff > 0)
diff_calc = EthDifficultyCalculator(
cutoff=cutoff, diff_factor=diff_factor, minimum_diff=min_diff
)
self.diff_calc = diff_calc
self.raw_db = env.db
self.db = RootDb(
self.raw_db,
env.quark_chain_config,
count_minor_blocks=env.cluster_config.ENABLE_TRANSACTION_HISTORY,
)
# header hash -> [coinbase address] during previous blocks (ascending)
self.coinbase_addr_cache = LRUCache(maxsize=128)
self.tip = self.db.get_tip_header() # type: RootBlockHeader
if self.tip:
Logger.info(
"Recovered root state with tip height {}".format(self.tip.height)
)
else:
self.tip = self.__create_genesis_block()
Logger.info("Created genesis root block")
def __init__(self, env, diff_calc=None):
self.env = env
self.root_config = env.quark_chain_config.ROOT
if not diff_calc:
cutoff = self.root_config.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME
diff_factor = self.root_config.DIFFICULTY_ADJUSTMENT_FACTOR
min_diff = self.root_config.GENESIS.DIFFICULTY
check(cutoff > 0 and diff_factor > 0 and min_diff > 0)
diff_calc = EthDifficultyCalculator(cutoff=cutoff,
diff_factor=diff_factor,
minimum_diff=min_diff)
self.diff_calc = diff_calc
self.raw_db = env.db
self.db = RootDb(
self.raw_db,
env.quark_chain_config,
count_minor_blocks=env.cluster_config.ENABLE_TRANSACTION_HISTORY,
)
persisted_tip = self.db.get_tip_header()
if persisted_tip:
self.tip = persisted_tip
Logger.info("Recovered root state with tip height {}".format(
self.tip.height))
else:
self.__create_genesis_block()
Logger.info("Created genesis root block")
def put_root_block_index(self, block):
block_hash = block.header.get_hash()
self.db.put(b"ri_%d" % block.header.height, block_hash)
if not self.count_minor_blocks:
return
# Count minor blocks by miner address
if block.header.height > 0:
shard_recipient_cnt = self.get_block_count(block.header.height - 1)
else:
shard_recipient_cnt = dict()
for header in block.minor_block_header_list:
full_shard_id = header.branch.get_full_shard_id()
recipient = header.coinbase_address.recipient.hex()
old_count = shard_recipient_cnt.get(full_shard_id,
dict()).get(recipient, 0)
new_count = old_count + 1
shard_recipient_cnt.setdefault(full_shard_id,
dict())[recipient] = new_count
block_id = header.get_hash() + full_shard_id.to_bytes(
4, byteorder="big")
self.db.put(b"m_r_" + block_id, block_hash)
for full_shard_id, r_c in shard_recipient_cnt.items():
data = bytearray()
for recipient, count in r_c.items():
data.extend(bytes.fromhex(recipient))
data.extend(count.to_bytes(4, "big"))
check(len(data) % 24 == 0)
self.db.put(b"count_%d_%d" % (full_shard_id, block.header.height),
data)
def shutdown_clusters(cluster_list, expect_aborted_rpc_count=0):
loop = asyncio.get_event_loop()
# allow pending RPCs to finish to avoid annoying connection reset error messages
loop.run_until_complete(asyncio.sleep(0.1))
for cluster in cluster_list:
# Shutdown simple network first
cluster.network.shutdown()
# Sleep 0.1 so that DESTROY_CLUSTER_PEER_ID command could be processed
loop.run_until_complete(asyncio.sleep(0.1))
for cluster in cluster_list:
for slave in cluster.slave_list:
slave.master.close()
loop.run_until_complete(slave.get_shutdown_future())
for slave in cluster.master.slave_pool:
slave.close()
cluster.master.shutdown()
loop.run_until_complete(cluster.master.get_shutdown_future())
check(expect_aborted_rpc_count == AbstractConnection.aborted_rpc_count)
def from_dict(cls, d, chains: Optional[List[ChainConfig]] = None):
config = super().from_dict(d)
# bail if both full shard ID list and chain mask list exist
shard_ids = getattr(config, "FULL_SHARD_ID_LIST", None)
chain_mask = getattr(config, "CHAIN_MASK_LIST", None)
if shard_ids and chain_mask:
raise ValueError(
"Can only have either FULL_SHARD_ID_LIST or CHAIN_MASK_LIST")
elif shard_ids:
# parse from hex to int
config.FULL_SHARD_ID_LIST = [
int(h, 16) for h in config.FULL_SHARD_ID_LIST
]
elif chain_mask:
if chains is None:
raise ValueError(
"Can't handle legacy CHAIN_MASK_LIST without chain configs"
)
# a simple way to be backward compatible with hard-coded shard ID
# e.g. chain mask 4 => 0x00000001, 0x00040001
# note that this only works if every chain has 1 shard only
check(all(chain.SHARD_SIZE == 1 for chain in chains))
for m in chain_mask:
bit_mask = (1 << (int_left_most_bit(m) - 1)) - 1
config.FULL_SHARD_ID_LIST = [
int("0x{:04x}0001".format(chain_id), 16)
for chain_id in range(len(chains))
if chain_id & bit_mask == m & bit_mask
]
delattr(config, "CHAIN_MASK_LIST")
else:
raise ValueError(
"Missing FULL_SHARD_ID_LIST (or CHAIN_MASK_LIST as legacy config)"
)
return config
def __create_from_args_internal():
check(
is_p2(args.num_shards_per_chain),
"--num_shards_per_chain must be power of 2",
)
check(is_p2(args.num_slaves), "--num_slaves must be power of 2")
config = ClusterConfig()
config.LOG_LEVEL = args.log_level
config.DB_PATH_ROOT = args.db_path_root
config.P2P_PORT = args.p2p_port
config.JSON_RPC_PORT = args.json_rpc_port
config.PRIVATE_JSON_RPC_PORT = args.json_rpc_private_port
config.CLEAN = args.clean
config.START_SIMULATED_MINING = args.start_simulated_mining
config.ENABLE_TRANSACTION_HISTORY = args.enable_transaction_history
config.QUARKCHAIN.update(
args.num_chains,
args.num_shards_per_chain,
args.root_block_interval_sec,
args.minor_block_interval_sec,
)
config.QUARKCHAIN.NETWORK_ID = args.network_id
config.GENESIS_DIR = args.genesis_dir
config.MONITORING.KAFKA_REST_ADDRESS = args.monitoring_kafka_rest_address
if args.p2p:
config.SIMPLE_NETWORK = None
config.P2P = P2PConfig()
# p2p module
config.P2P.BOOT_NODES = args.bootnodes
config.P2P.PRIV_KEY = args.privkey
config.P2P.MAX_PEERS = args.max_peers
config.P2P.UPNP = args.upnp
else:
config.P2P = None
config.SIMPLE_NETWORK = SimpleNetworkConfig()
config.SIMPLE_NETWORK.BOOTSTRAP_HOST = (
args.simple_network_bootstrap_host)
config.SIMPLE_NETWORK.BOOTSTRAP_PORT = (
args.simple_network_bootstrap_port)
config.SLAVE_LIST = []
for i in range(args.num_slaves):
slave_config = SlaveConfig()
slave_config.PORT = args.port_start + i
slave_config.ID = "S{}".format(i)
slave_config.SHARD_MASK_LIST = [ShardMask(i | args.num_slaves)]
config.SLAVE_LIST.append(slave_config)
fd, config.json_filepath = tempfile.mkstemp()
with os.fdopen(fd, "w") as tmp:
tmp.write(config.to_json())
return config
def calculate_diff_with_parent(self, parent, create_time):
# TODO: support uncle
check(not self.check_uncle)
check(parent.create_time < create_time)
sign = max(1 - (create_time - parent.create_time) // self.cutoff, -99)
offset = parent.difficulty // self.diff_factor
return int(max(parent.difficulty + offset * sign, self.minimum_diff))
async def handle_metadata_and_raw_data(self, metadata, raw_data):
forward_conn = self.get_connection_to_forward(metadata)
if forward_conn:
check(self.validate_connection(forward_conn))
return forward_conn.write_raw_data(
self.get_metadata_to_forward(metadata), raw_data)
await super().handle_metadata_and_raw_data(metadata, raw_data)
async def handle_sync_minor_block_list_request(self, req):
""" Raises on error"""
async def __download_blocks(block_hash_list):
op, resp, rpc_id = await peer_shard_conn.write_rpc_request(
CommandOp.GET_MINOR_BLOCK_LIST_REQUEST,
GetMinorBlockListRequest(block_hash_list),
)
return resp.minor_block_list
shard = self.shards.get(req.branch, None)
if not shard:
return SyncMinorBlockListResponse(error_code=errno.EBADMSG)
peer_shard_conn = shard.peers.get(req.cluster_peer_id, None)
if not peer_shard_conn:
return SyncMinorBlockListResponse(error_code=errno.EBADMSG)
BLOCK_BATCH_SIZE = 100
block_hash_list = req.minor_block_hash_list
# empty
if not block_hash_list:
return SyncMinorBlockListResponse(error_code=0)
try:
while len(block_hash_list) > 0:
blocks_to_download = block_hash_list[:BLOCK_BATCH_SIZE]
try:
block_chain = await asyncio.wait_for(
__download_blocks(blocks_to_download), TIMEOUT)
except asyncio.TimeoutError as e:
Logger.info(
"[{}] sync request from master failed due to timeout".
format(req.branch.get_full_shard_id()))
raise e
Logger.info(
"[{}] sync request from master, downloaded {} blocks ({} - {})"
.format(
req.branch.get_full_shard_id(),
len(block_chain),
block_chain[0].header.height,
block_chain[-1].header.height,
))
check(len(block_chain) == len(blocks_to_download))
add_block_success = await self.slave_server.add_block_list_for_sync(
block_chain)
if not add_block_success:
raise RuntimeError(
"Failed to add minor blocks for syncing root block")
block_hash_list = block_hash_list[BLOCK_BATCH_SIZE:]
branch = block_chain[0].header.branch
shard = self.slave_server.shards.get(branch, None)
check(shard is not None)
return SyncMinorBlockListResponse(
error_code=0, shard_stats=shard.state.get_shard_stats())
except Exception:
Logger.error_exception()
return SyncMinorBlockListResponse(error_code=1)
async def batch_broadcast_xshard_tx_list(
self,
block_hash_to_xshard_list_and_prev_root_height: Dict[bytes, Tuple[List, int]],
source_branch: Branch,
):
branch_to_add_xshard_tx_list_request_list = dict()
for (
block_hash,
x_shard_list_and_prev_root_height,
) in block_hash_to_xshard_list_and_prev_root_height.items():
xshard_tx_list = x_shard_list_and_prev_root_height[0]
prev_root_height = x_shard_list_and_prev_root_height[1]
branch_to_add_xshard_tx_list_request = self.__get_branch_to_add_xshard_tx_list_request(
block_hash, xshard_tx_list, prev_root_height
)
for branch, request in branch_to_add_xshard_tx_list_request.items():
if branch == source_branch or not is_neighbor(
branch,
source_branch,
len(
self.env.quark_chain_config.get_initialized_full_shard_ids_before_root_height(
prev_root_height
)
),
):
check(
len(request.tx_list.tx_list) == 0,
"there shouldn't be xshard list for non-neighbor shard ({} -> {})".format(
source_branch.value, branch.value
),
)
continue
branch_to_add_xshard_tx_list_request_list.setdefault(branch, []).append(
request
)
rpc_futures = []
for branch, request_list in branch_to_add_xshard_tx_list_request_list.items():
if branch in self.shards:
for request in request_list:
self.shards[
branch
].state.add_cross_shard_tx_list_by_minor_block_hash(
request.minor_block_hash, request.tx_list
)
batch_request = BatchAddXshardTxListRequest(request_list)
for (
slave_conn
) in self.slave_connection_manager.get_connections_by_full_shard_id(
branch.get_full_shard_id()
):
future = slave_conn.write_rpc_request(
ClusterOp.BATCH_ADD_XSHARD_TX_LIST_REQUEST, batch_request
)
rpc_futures.append(future)
responses = await asyncio.gather(*rpc_futures)
check(all([response.error_code == 0 for _, response, _ in responses]))
def get_transaction_by_hash(
self, tx_hash) -> Tuple[Optional[MinorBlock], Optional[int]]:
result = self.db.get(b"txindex_" + tx_hash, None)
if not result:
return None, None
check(len(result) == 8)
block_height = int.from_bytes(result[:4], "big")
index = int.from_bytes(result[4:], "big")
return self.get_minor_block_by_height(block_height), index
async def add_root_block(self, root_block: RootBlock):
check(root_block.header.height >= self.genesis_root_height)
if root_block.header.height > self.genesis_root_height:
return self.state.add_root_block(root_block)
# this happens when there is a root chain fork
if root_block.header.height == self.genesis_root_height:
await self.__init_genesis_state(root_block)
async def add_block_list_for_sync(self, block_list):
""" Add blocks in batch to reduce RPCs. Will NOT broadcast to peers.
Returns true if blocks are successfully added. False on any error.
"""
if not block_list:
return True
branch = block_list[0].header.branch
shard = self.shards.get(branch, None)
check(shard is not None)
return await shard.add_block_list_for_sync(block_list)
async def send_minor_block_header_to_master(self, minor_block_header,
tx_count, x_shard_tx_count,
shard_stats):
""" Update master that a minor block has been appended successfully """
request = AddMinorBlockHeaderRequest(minor_block_header, tx_count,
x_shard_tx_count, shard_stats)
_, resp, _ = await self.master.write_rpc_request(
ClusterOp.ADD_MINOR_BLOCK_HEADER_REQUEST, request)
check(resp.error_code == 0)
self.artificial_tx_config = resp.artificial_tx_config
async def add_block_list_for_sync(self, block_list):
""" Add blocks in batch to reduce RPCs. Will NOT broadcast to peers.
Returns true if blocks are successfully added. False on any error.
Additionally, returns list of coinbase_amount_map for each block
This function only adds blocks to local and propagate xshard list to other shards.
It does NOT notify master because the master should already have the minor header list,
and will add them once this function returns successfully.
"""
coinbase_amount_list = []
if not block_list:
return True, coinbase_amount_list
existing_add_block_futures = []
block_hash_to_x_shard_list = dict()
for block in block_list:
check(
block.header.branch.get_full_shard_id() == self.full_shard_id)
block_hash = block.header.get_hash()
try:
xshard_list, coinbase_amount_map = self.state.add_block(
block, skip_if_too_old=False)
# coinbase_amount_map may be None if the block exists
# adding the block header one since the block is already validated.
coinbase_amount_list.append(block.header.coinbase_amount_map)
except Exception as e:
Logger.error_exception()
return False, coinbase_amount_list
# block already existed in local shard state
# but might not have been propagated to other shards and master
# let's make sure all the shards and master got it before return
if xshard_list is None:
future = self.add_block_futures.get(block_hash, None)
if future:
existing_add_block_futures.append(future)
else:
prev_root_height = self.state.db.get_root_block_header_by_hash(
block.header.hash_prev_root_block).height
block_hash_to_x_shard_list[block_hash] = (xshard_list,
prev_root_height)
self.add_block_futures[block_hash] = self.loop.create_future()
await self.slave.batch_broadcast_xshard_tx_list(
block_hash_to_x_shard_list, block_list[0].header.branch)
for block_hash in block_hash_to_x_shard_list.keys():
self.add_block_futures[block_hash].set_result(None)
del self.add_block_futures[block_hash]
await asyncio.gather(*existing_add_block_futures)
return True, coinbase_amount_list
def mine_next_minor_block(self, ts):
check(self.minor_block_to_mine is None)
m_block = self.get_next_minor_block_to_mine(ts)
if m_block is None:
return
self.minor_block_to_mine = m_block
self.scheduler.schedule_after(m_block.mined_time - ts,
self.mine_minor_block,
self.minor_block_to_mine)
async def get_work(self, branch: Branch) -> Optional[MiningWork]:
try:
shard = self.shards[branch]
work, block = await shard.miner.get_work()
if shard.state.shard_config.POSW_CONFIG.ENABLED:
check(isinstance(block, MinorBlock))
diff = shard.state.posw_diff_adjust(block)
work = MiningWork(work.hash, work.height, diff)
return work
except Exception:
Logger.log_exception()
return None
def get_block_count(self, root_height, shard_size):
"""Returns a list(dict(miner_recipient, block_count)) of size shard_size"""
shard_recipient_cnt = [dict() for _ in range(shard_size)]
if not self.count_minor_blocks:
return shard_recipient_cnt
for shard in range(shard_size):
data = self.db.get(b"count_%d_%d" % (shard, root_height))
check(len(data) % 24 == 0)
for i in range(0, len(data), 24):
recipient = data[i:i + 20].hex()
count = int.from_bytes(data[i + 20:i + 24], "big")
shard_recipient_cnt[shard][recipient] = count
return shard_recipient_cnt
def is_neighbor(b1: Branch, b2: Branch):
"""A naive algorithm to decide neighbor relationship
Two shards are neighbor iff there is only 1 bit difference in their shard ids.
This only applies if there are more than 32 shards in the network.
Otherwise all shards are neighbor to each other.
TODO: a better algorithm
"""
check(b1.get_shard_size() == b2.get_shard_size())
check(b1.get_shard_id() != b2.get_shard_id())
if b1.get_shard_size() <= 32:
return True
return is_p2(abs(b1.get_shard_id() - b2.get_shard_id()))
def get_posw_info(
config: POSWConfig,
header: Header,
stakes: int,
block_cnt: Dict[bytes, int],
stake_per_block: Optional[int] = None,
signer: Optional[bytes] = None,
) -> Optional[PoSWInfo]:
if (not (config.ENABLED and header.create_time >= config.ENABLE_TIMESTAMP)
or header.height == 0):
return None
# evaluate stakes before the to-be-added block
coinbase_recipient = header.coinbase_address.recipient
required_stakes_per_block = stake_per_block or config.TOTAL_STAKE_PER_BLOCK
block_threshold = min(config.WINDOW_SIZE,
stakes // required_stakes_per_block)
cnt = block_cnt.get(coinbase_recipient, 0)
diff = header.difficulty
ret = lambda success: PoSWInfo(
diff // config.get_diff_divider(header.create_time)
if success else diff,
block_threshold,
# mined blocks should include current one, assuming success
posw_mined_blocks=cnt + 1,
)
# fast path
if block_threshold == 0:
return ret(False)
# need to check signature if signer is specified. only applies for root chain
if signer:
check(isinstance(header, RootBlockHeader))
if signer == bytes(20):
return ret(False)
block_sig = Signature(header.signature)
try:
pubk = block_sig.recover_public_key_from_msg_hash(
header.get_hash_for_mining())
except BadSignature:
return ret(False)
if pubk.to_canonical_address() != signer:
return ret(False)
return ret(cnt < block_threshold)
def get_test_env(
genesis_account=Address.create_empty_account(),
genesis_minor_quarkash=0,
shard_size=2,
genesis_root_heights=None,
remote_mining=False,
):
env = DEFAULT_ENV.copy()
env.db = InMemoryDb()
env.set_network_id(1234567890)
env.cluster_config = ClusterConfig()
env.quark_chain_config.update(shard_size, 10, 1)
if remote_mining:
env.quark_chain_config.ROOT.CONSENSUS_CONFIG.REMOTE_MINE = True
env.quark_chain_config.ROOT.CONSENSUS_TYPE = ConsensusType.POW_SHA3SHA3
env.quark_chain_config.ROOT.GENESIS.DIFFICULTY = 10
env.quark_chain_config.ROOT.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME = 40
env.quark_chain_config.ROOT.DIFFICULTY_ADJUSTMENT_FACTOR = 1024
if genesis_root_heights:
check(len(genesis_root_heights) == shard_size)
for shard_id in range(shard_size):
shard = env.quark_chain_config.SHARD_LIST[shard_id]
shard.GENESIS.ROOT_HEIGHT = genesis_root_heights[shard_id]
# fund genesis account in all shards
for i, shard in enumerate(env.quark_chain_config.SHARD_LIST):
addr = genesis_account.address_in_shard(i).serialize().hex()
shard.GENESIS.ALLOC[addr] = genesis_minor_quarkash
shard.CONSENSUS_CONFIG.REMOTE_MINE = remote_mining
shard.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME = 7
shard.DIFFICULTY_ADJUSTMENT_FACTOR = 512
if remote_mining:
shard.CONSENSUS_TYPE = ConsensusType.POW_SHA3SHA3
shard.GENESIS.DIFFICULTY = 10
env.quark_chain_config.SKIP_MINOR_DIFFICULTY_CHECK = True
env.quark_chain_config.SKIP_ROOT_DIFFICULTY_CHECK = True
env.cluster_config.ENABLE_TRANSACTION_HISTORY = True
env.cluster_config.DB_PATH_ROOT = ""
check(env.cluster_config.use_mem_db())
return env
