async def __run_sync(self): if self.__has_block_hash(self.header.get_hash()): return # descending height block_header_chain = [self.header] # TODO: Stop if too many headers to revert while not self.__has_block_hash( block_header_chain[-1].hash_prev_minor_block): block_hash = block_header_chain[-1].hash_prev_minor_block height = block_header_chain[-1].height - 1 if self.shard_state.header_tip.height - height > self.max_staleness: Logger.warning( "[{}] abort syncing due to forking at very old block {} << {}" .format( self.header.branch.get_shard_id(), height, self.shard_state.header_tip.height, )) return if not self.shard_state.db.contain_root_block_by_hash( block_header_chain[-1].hash_prev_root_block): return Logger.info("[{}] downloading headers from {} {}".format( self.shard_state.branch.get_shard_id(), height, block_hash.hex())) block_header_list = await asyncio.wait_for( self.__download_block_headers(block_hash), TIMEOUT) Logger.info("[{}] downloaded {} headers from peer".format( self.shard_state.branch.get_shard_id(), len(block_header_list))) if not self.__validate_block_headers(block_header_list): # TODO: tag bad peer return self.shard_conn.close_with_error( "Bad peer sending discontinuing block headers") for header in block_header_list: if self.__has_block_hash(header.get_hash()): break block_header_chain.append(header) # ascending height block_header_chain.reverse() while len(block_header_chain) > 0: block_chain = await asyncio.wait_for( self.__download_blocks(block_header_chain[:100]), TIMEOUT) Logger.info("[{}] downloaded {} blocks from peer".format( self.shard_state.branch.get_shard_id(), len(block_chain))) check(len(block_chain) == len(block_header_chain[:100])) for block in block_chain: # Stop if the block depends on an unknown root block # TODO: move this check to early stage to avoid downloading unnecessary headers if not self.shard_state.db.contain_root_block_by_hash( block.header.hash_prev_root_block): return await self.shard.add_block(block) block_header_chain.pop(0)
def mine_next_root_block(self, ts): check(self.root_block_to_mine is None) if (len(self.pending_minor_block_queue) == 0 or self.pending_minor_block_queue[0].root_block.block_height + self.args.rprevious > self.get_root_block_tip().block_height + 1): # The root block is not able to mine. Will wait until a minor block is produced. # TODO: the root block may also mine an null minor block with reduced coinbase reward return self.root_block_to_mine = RootBlock( block_height=self.get_root_block_tip().block_height + 1, minor_block_list=[], # to be fill once mined mining_time=ts, mined_time=None, ) self.scheduler.schedule_after( get_next_interval(ROOT_BLOCK_INTERVAL_SEC), self.mine_root_block, self.root_block_to_mine, ) if self.args.verbose >= 1: print("%0.2f: root_block %d mining ..." % (ts, self.root_block_to_mine.block_height))
def get_test_env( genesis_account=Address.create_empty_account(), genesis_quarkash=0, genesis_minor_quarkash=0, shard_size=2, genesis_root_heights=None, ): env = DEFAULT_ENV.copy() env.db = InMemoryDb() env.set_network_id(1234567890) env.cluster_config = ClusterConfig() env.quark_chain_config.update(shard_size, 1, 1) env.quark_chain_config.TESTNET_MASTER_ADDRESS = genesis_account.serialize( ).hex() if genesis_root_heights: check(len(genesis_root_heights) == shard_size) for shard_id in range(shard_size): env.quark_chain_config.SHARD_LIST[ shard_id].GENESIS.ROOT_HEIGHT = genesis_root_heights[shard_id] # fund genesis account in all shards for i, shard in enumerate(env.quark_chain_config.SHARD_LIST): shard.GENESIS.ALLOC[genesis_account.address_in_shard( i).serialize().hex()] = genesis_minor_quarkash env.quark_chain_config.SKIP_MINOR_DIFFICULTY_CHECK = True env.quark_chain_config.SKIP_ROOT_DIFFICULTY_CHECK = True env.cluster_config.ENABLE_TRANSACTION_HISTORY = True env.cluster_config.DB_PATH_ROOT = "" check(env.cluster_config.use_mem_db()) return env
def create_minor_block(self, root_block: RootBlock, full_shard_id: int, evm_state: EvmState) -> MinorBlock: """ Create genesis block for shard. Genesis block's hash_prev_root_block is set to the genesis root block. Genesis state will be committed to the given evm_state. Based on ALLOC, genesis_token will be added to initial accounts. """ branch = Branch(full_shard_id) shard_config = self._qkc_config.shards[full_shard_id] genesis = shard_config.GENESIS for address_hex, alloc_amount in genesis.ALLOC.items(): address = Address.create_from(bytes.fromhex(address_hex)) check( self._qkc_config.get_full_shard_id_by_full_shard_key( address.full_shard_key) == full_shard_id) evm_state.full_shard_key = address.full_shard_key if isinstance(alloc_amount, dict): for k, v in alloc_amount.items(): evm_state.delta_token_balance(address.recipient, token_id_encode(k), v) else: evm_state.delta_token_balance(address.recipient, self._qkc_config.genesis_token, alloc_amount) evm_state.commit() meta = MinorBlockMeta( hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT), hash_evm_state_root=evm_state.trie.root_hash, ) local_fee_rate = 1 - self._qkc_config.reward_tax_rate # type: Fraction coinbase_tokens = { self._qkc_config.genesis_token: shard_config.COINBASE_AMOUNT * local_fee_rate.numerator // local_fee_rate.denominator } coinbase_address = Address.create_empty_account(full_shard_id) header = MinorBlockHeader( version=genesis.VERSION, height=genesis.HEIGHT, branch=branch, hash_prev_minor_block=bytes.fromhex(genesis.HASH_PREV_MINOR_BLOCK), hash_prev_root_block=root_block.header.get_hash(), evm_gas_limit=genesis.GAS_LIMIT, hash_meta=sha3_256(meta.serialize()), coinbase_amount_map=TokenBalanceMap(coinbase_tokens), coinbase_address=coinbase_address, create_time=genesis.TIMESTAMP, difficulty=genesis.DIFFICULTY, extra_data=bytes.fromhex(genesis.EXTRA_DATA), ) return ( MinorBlock(header=header, meta=meta, tx_list=[]), TokenBalanceMap(coinbase_tokens), )
def __get_branch_to_add_xshard_tx_list_request(self, block_hash, xshard_tx_list, prev_root_height): xshard_map = dict( ) # type: Dict[Branch, List[CrossShardTransactionDeposit]] # only broadcast to the shards that have been initialized initialized_shard_ids = self.env.quark_chain_config.get_initialized_shard_ids_before_root_height( prev_root_height) for shard_id in initialized_shard_ids: branch = Branch.create(self.__get_shard_size(), shard_id) xshard_map[branch] = [] for xshard_tx in xshard_tx_list: shard_id = xshard_tx.to_address.get_shard_id( self.__get_shard_size()) branch = Branch.create(self.__get_shard_size(), shard_id) check(branch in xshard_map) xshard_map[branch].append(xshard_tx) branch_to_add_xshard_tx_list_request = ( dict()) # type: Dict[Branch, AddXshardTxListRequest] for branch, tx_list in xshard_map.items(): cross_shard_tx_list = CrossShardTransactionList(tx_list) request = AddXshardTxListRequest(branch, block_hash, cross_shard_tx_list) branch_to_add_xshard_tx_list_request[branch] = request return branch_to_add_xshard_tx_list_request
def put_root_block_index(self, block): self.db.put(b"ri_%d" % block.header.height, block.header.get_hash()) if not self.count_minor_blocks: return # Count minor blocks by miner address shard_size = block.header.shard_info.get_shard_size() if block.header.height > 0: shard_recipient_cnt = self.get_block_count(block.header.height - 1, shard_size) else: shard_recipient_cnt = [dict() for _ in range(shard_size)] for header in block.minor_block_header_list: shard = header.branch.get_shard_id() recipient = header.coinbase_address.recipient.hex() shard_recipient_cnt[shard][recipient] = ( shard_recipient_cnt[shard].get(recipient, 0) + 1) for shard, r_c in enumerate(shard_recipient_cnt): data = bytearray() for recipient, count in r_c.items(): data.extend(bytes.fromhex(recipient)) data.extend(count.to_bytes(4, "big")) check(len(data) % 24 == 0) self.db.put(b"count_%d_%d" % (shard, block.header.height), data)
async def broadcast_xshard_tx_list(self, block, xshard_tx_list, prev_root_height): """ Broadcast x-shard transactions to their recipient shards """ block_hash = block.header.get_hash() branch_to_add_xshard_tx_list_request = self.__get_branch_to_add_xshard_tx_list_request( block_hash, xshard_tx_list, prev_root_height) rpc_futures = [] for branch, request in branch_to_add_xshard_tx_list_request.items(): if branch == block.header.branch or not is_neighbor( block.header.branch, branch): continue if branch in self.shards: self.shards[ branch].state.add_cross_shard_tx_list_by_minor_block_hash( block_hash, request.tx_list) for slave_conn in self.slave_connection_manager.get_connections_by_shard( branch.get_shard_id()): future = slave_conn.write_rpc_request( ClusterOp.ADD_XSHARD_TX_LIST_REQUEST, request) rpc_futures.append(future) responses = await asyncio.gather(*rpc_futures) check(all([response.error_code == 0 for _, response, _ in responses]))
def __init__( self, num_cluster, genesis_account=Address.create_empty_account(), chain_size=2, shard_size=2, num_slaves=None, genesis_root_heights=None, remote_mining=False, small_coinbase=False, loadtest_accounts=None, connect=True, should_set_gas_price_limit=False, mblock_coinbase_amount=None, genesis_minor_quarkash=1000000, ): self.num_cluster = num_cluster self.genesis_account = genesis_account self.chain_size = chain_size self.shard_size = shard_size self.num_slaves = num_slaves if num_slaves else chain_size self.genesis_root_heights = genesis_root_heights self.remote_mining = remote_mining self.small_coinbase = small_coinbase self.loadtest_accounts = loadtest_accounts self.connect = connect self.should_set_gas_price_limit = should_set_gas_price_limit self.mblock_coinbase_amount = mblock_coinbase_amount self.genesis_minor_quarkash = genesis_minor_quarkash check(is_p2(self.num_slaves)) check(is_p2(self.shard_size))
def get_next_minor_block_to_mine(self, ts): tip = self.get_minor_block_tip() # Produce block in this epoch if the epoch has space if tip.root_index < MINOR_BLOCK_PER_ROOT_BLOCK - 1: check(ts == tip.mined_time) return MinorBlock( root_block=tip.root_block, root_index=tip.root_index + 1, block_height=tip.block_height + 1, mined_time=ts + MINOR_BLOCK_INTERVAL_SEC, ) # Produce block in next epoch if a root block is available root_height_to_confirm = tip.root_block.block_height + 1 if root_height_to_confirm <= self.get_root_block_tip().block_height: root_block = self.root_block_list[root_height_to_confirm] check(ts == max(tip.mined_time, root_block.mined_time)) return MinorBlock( root_block=root_block, root_index=0, block_height=tip.block_height + 1, mined_time=ts + MINOR_BLOCK_INTERVAL_SEC, ) # Unable to find a root block to produce the minor block return None
def __init__(self, env, diff_calc=None): self.env = env self.root_config = env.quark_chain_config.ROOT if not diff_calc: cutoff = self.root_config.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME diff_factor = self.root_config.DIFFICULTY_ADJUSTMENT_FACTOR min_diff = self.root_config.GENESIS.DIFFICULTY check(cutoff > 0 and diff_factor > 0 and min_diff > 0) diff_calc = EthDifficultyCalculator( cutoff=cutoff, diff_factor=diff_factor, minimum_diff=min_diff ) self.diff_calc = diff_calc self.raw_db = env.db self.db = RootDb( self.raw_db, env.quark_chain_config, count_minor_blocks=env.cluster_config.ENABLE_TRANSACTION_HISTORY, ) # header hash -> [coinbase address] during previous blocks (ascending) self.coinbase_addr_cache = LRUCache(maxsize=128) self.tip = self.db.get_tip_header() # type: RootBlockHeader if self.tip: Logger.info( "Recovered root state with tip height {}".format(self.tip.height) ) else: self.tip = self.__create_genesis_block() Logger.info("Created genesis root block")
def __init__(self, env, diff_calc=None): self.env = env self.root_config = env.quark_chain_config.ROOT if not diff_calc: cutoff = self.root_config.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME diff_factor = self.root_config.DIFFICULTY_ADJUSTMENT_FACTOR min_diff = self.root_config.GENESIS.DIFFICULTY check(cutoff > 0 and diff_factor > 0 and min_diff > 0) diff_calc = EthDifficultyCalculator(cutoff=cutoff, diff_factor=diff_factor, minimum_diff=min_diff) self.diff_calc = diff_calc self.raw_db = env.db self.db = RootDb( self.raw_db, env.quark_chain_config, count_minor_blocks=env.cluster_config.ENABLE_TRANSACTION_HISTORY, ) persisted_tip = self.db.get_tip_header() if persisted_tip: self.tip = persisted_tip Logger.info("Recovered root state with tip height {}".format( self.tip.height)) else: self.__create_genesis_block() Logger.info("Created genesis root block")
def put_root_block_index(self, block): block_hash = block.header.get_hash() self.db.put(b"ri_%d" % block.header.height, block_hash) if not self.count_minor_blocks: return # Count minor blocks by miner address if block.header.height > 0: shard_recipient_cnt = self.get_block_count(block.header.height - 1) else: shard_recipient_cnt = dict() for header in block.minor_block_header_list: full_shard_id = header.branch.get_full_shard_id() recipient = header.coinbase_address.recipient.hex() old_count = shard_recipient_cnt.get(full_shard_id, dict()).get(recipient, 0) new_count = old_count + 1 shard_recipient_cnt.setdefault(full_shard_id, dict())[recipient] = new_count block_id = header.get_hash() + full_shard_id.to_bytes( 4, byteorder="big") self.db.put(b"m_r_" + block_id, block_hash) for full_shard_id, r_c in shard_recipient_cnt.items(): data = bytearray() for recipient, count in r_c.items(): data.extend(bytes.fromhex(recipient)) data.extend(count.to_bytes(4, "big")) check(len(data) % 24 == 0) self.db.put(b"count_%d_%d" % (full_shard_id, block.header.height), data)
def shutdown_clusters(cluster_list, expect_aborted_rpc_count=0): loop = asyncio.get_event_loop() # allow pending RPCs to finish to avoid annoying connection reset error messages loop.run_until_complete(asyncio.sleep(0.1)) for cluster in cluster_list: # Shutdown simple network first cluster.network.shutdown() # Sleep 0.1 so that DESTROY_CLUSTER_PEER_ID command could be processed loop.run_until_complete(asyncio.sleep(0.1)) for cluster in cluster_list: for slave in cluster.slave_list: slave.master.close() loop.run_until_complete(slave.get_shutdown_future()) for slave in cluster.master.slave_pool: slave.close() cluster.master.shutdown() loop.run_until_complete(cluster.master.get_shutdown_future()) check(expect_aborted_rpc_count == AbstractConnection.aborted_rpc_count)
def from_dict(cls, d, chains: Optional[List[ChainConfig]] = None): config = super().from_dict(d) # bail if both full shard ID list and chain mask list exist shard_ids = getattr(config, "FULL_SHARD_ID_LIST", None) chain_mask = getattr(config, "CHAIN_MASK_LIST", None) if shard_ids and chain_mask: raise ValueError( "Can only have either FULL_SHARD_ID_LIST or CHAIN_MASK_LIST") elif shard_ids: # parse from hex to int config.FULL_SHARD_ID_LIST = [ int(h, 16) for h in config.FULL_SHARD_ID_LIST ] elif chain_mask: if chains is None: raise ValueError( "Can't handle legacy CHAIN_MASK_LIST without chain configs" ) # a simple way to be backward compatible with hard-coded shard ID # e.g. chain mask 4 => 0x00000001, 0x00040001 # note that this only works if every chain has 1 shard only check(all(chain.SHARD_SIZE == 1 for chain in chains)) for m in chain_mask: bit_mask = (1 << (int_left_most_bit(m) - 1)) - 1 config.FULL_SHARD_ID_LIST = [ int("0x{:04x}0001".format(chain_id), 16) for chain_id in range(len(chains)) if chain_id & bit_mask == m & bit_mask ] delattr(config, "CHAIN_MASK_LIST") else: raise ValueError( "Missing FULL_SHARD_ID_LIST (or CHAIN_MASK_LIST as legacy config)" ) return config
def __create_from_args_internal(): check( is_p2(args.num_shards_per_chain), "--num_shards_per_chain must be power of 2", ) check(is_p2(args.num_slaves), "--num_slaves must be power of 2") config = ClusterConfig() config.LOG_LEVEL = args.log_level config.DB_PATH_ROOT = args.db_path_root config.P2P_PORT = args.p2p_port config.JSON_RPC_PORT = args.json_rpc_port config.PRIVATE_JSON_RPC_PORT = args.json_rpc_private_port config.CLEAN = args.clean config.START_SIMULATED_MINING = args.start_simulated_mining config.ENABLE_TRANSACTION_HISTORY = args.enable_transaction_history config.QUARKCHAIN.update( args.num_chains, args.num_shards_per_chain, args.root_block_interval_sec, args.minor_block_interval_sec, ) config.QUARKCHAIN.NETWORK_ID = args.network_id config.GENESIS_DIR = args.genesis_dir config.MONITORING.KAFKA_REST_ADDRESS = args.monitoring_kafka_rest_address if args.p2p: config.SIMPLE_NETWORK = None config.P2P = P2PConfig() # p2p module config.P2P.BOOT_NODES = args.bootnodes config.P2P.PRIV_KEY = args.privkey config.P2P.MAX_PEERS = args.max_peers config.P2P.UPNP = args.upnp else: config.P2P = None config.SIMPLE_NETWORK = SimpleNetworkConfig() config.SIMPLE_NETWORK.BOOTSTRAP_HOST = ( args.simple_network_bootstrap_host) config.SIMPLE_NETWORK.BOOTSTRAP_PORT = ( args.simple_network_bootstrap_port) config.SLAVE_LIST = [] for i in range(args.num_slaves): slave_config = SlaveConfig() slave_config.PORT = args.port_start + i slave_config.ID = "S{}".format(i) slave_config.SHARD_MASK_LIST = [ShardMask(i | args.num_slaves)] config.SLAVE_LIST.append(slave_config) fd, config.json_filepath = tempfile.mkstemp() with os.fdopen(fd, "w") as tmp: tmp.write(config.to_json()) return config
def calculate_diff_with_parent(self, parent, create_time): # TODO: support uncle check(not self.check_uncle) check(parent.create_time < create_time) sign = max(1 - (create_time - parent.create_time) // self.cutoff, -99) offset = parent.difficulty // self.diff_factor return int(max(parent.difficulty + offset * sign, self.minimum_diff))
async def handle_metadata_and_raw_data(self, metadata, raw_data): forward_conn = self.get_connection_to_forward(metadata) if forward_conn: check(self.validate_connection(forward_conn)) return forward_conn.write_raw_data( self.get_metadata_to_forward(metadata), raw_data) await super().handle_metadata_and_raw_data(metadata, raw_data)
async def handle_sync_minor_block_list_request(self, req): """ Raises on error""" async def __download_blocks(block_hash_list): op, resp, rpc_id = await peer_shard_conn.write_rpc_request( CommandOp.GET_MINOR_BLOCK_LIST_REQUEST, GetMinorBlockListRequest(block_hash_list), ) return resp.minor_block_list shard = self.shards.get(req.branch, None) if not shard: return SyncMinorBlockListResponse(error_code=errno.EBADMSG) peer_shard_conn = shard.peers.get(req.cluster_peer_id, None) if not peer_shard_conn: return SyncMinorBlockListResponse(error_code=errno.EBADMSG) BLOCK_BATCH_SIZE = 100 block_hash_list = req.minor_block_hash_list # empty if not block_hash_list: return SyncMinorBlockListResponse(error_code=0) try: while len(block_hash_list) > 0: blocks_to_download = block_hash_list[:BLOCK_BATCH_SIZE] try: block_chain = await asyncio.wait_for( __download_blocks(blocks_to_download), TIMEOUT) except asyncio.TimeoutError as e: Logger.info( "[{}] sync request from master failed due to timeout". format(req.branch.get_full_shard_id())) raise e Logger.info( "[{}] sync request from master, downloaded {} blocks ({} - {})" .format( req.branch.get_full_shard_id(), len(block_chain), block_chain[0].header.height, block_chain[-1].header.height, )) check(len(block_chain) == len(blocks_to_download)) add_block_success = await self.slave_server.add_block_list_for_sync( block_chain) if not add_block_success: raise RuntimeError( "Failed to add minor blocks for syncing root block") block_hash_list = block_hash_list[BLOCK_BATCH_SIZE:] branch = block_chain[0].header.branch shard = self.slave_server.shards.get(branch, None) check(shard is not None) return SyncMinorBlockListResponse( error_code=0, shard_stats=shard.state.get_shard_stats()) except Exception: Logger.error_exception() return SyncMinorBlockListResponse(error_code=1)
async def batch_broadcast_xshard_tx_list( self, block_hash_to_xshard_list_and_prev_root_height: Dict[bytes, Tuple[List, int]], source_branch: Branch, ): branch_to_add_xshard_tx_list_request_list = dict() for ( block_hash, x_shard_list_and_prev_root_height, ) in block_hash_to_xshard_list_and_prev_root_height.items(): xshard_tx_list = x_shard_list_and_prev_root_height[0] prev_root_height = x_shard_list_and_prev_root_height[1] branch_to_add_xshard_tx_list_request = self.__get_branch_to_add_xshard_tx_list_request( block_hash, xshard_tx_list, prev_root_height ) for branch, request in branch_to_add_xshard_tx_list_request.items(): if branch == source_branch or not is_neighbor( branch, source_branch, len( self.env.quark_chain_config.get_initialized_full_shard_ids_before_root_height( prev_root_height ) ), ): check( len(request.tx_list.tx_list) == 0, "there shouldn't be xshard list for non-neighbor shard ({} -> {})".format( source_branch.value, branch.value ), ) continue branch_to_add_xshard_tx_list_request_list.setdefault(branch, []).append( request ) rpc_futures = [] for branch, request_list in branch_to_add_xshard_tx_list_request_list.items(): if branch in self.shards: for request in request_list: self.shards[ branch ].state.add_cross_shard_tx_list_by_minor_block_hash( request.minor_block_hash, request.tx_list ) batch_request = BatchAddXshardTxListRequest(request_list) for ( slave_conn ) in self.slave_connection_manager.get_connections_by_full_shard_id( branch.get_full_shard_id() ): future = slave_conn.write_rpc_request( ClusterOp.BATCH_ADD_XSHARD_TX_LIST_REQUEST, batch_request ) rpc_futures.append(future) responses = await asyncio.gather(*rpc_futures) check(all([response.error_code == 0 for _, response, _ in responses]))
def get_transaction_by_hash( self, tx_hash) -> Tuple[Optional[MinorBlock], Optional[int]]: result = self.db.get(b"txindex_" + tx_hash, None) if not result: return None, None check(len(result) == 8) block_height = int.from_bytes(result[:4], "big") index = int.from_bytes(result[4:], "big") return self.get_minor_block_by_height(block_height), index
async def add_root_block(self, root_block: RootBlock): check(root_block.header.height >= self.genesis_root_height) if root_block.header.height > self.genesis_root_height: return self.state.add_root_block(root_block) # this happens when there is a root chain fork if root_block.header.height == self.genesis_root_height: await self.__init_genesis_state(root_block)
async def add_block_list_for_sync(self, block_list): """ Add blocks in batch to reduce RPCs. Will NOT broadcast to peers. Returns true if blocks are successfully added. False on any error. """ if not block_list: return True branch = block_list[0].header.branch shard = self.shards.get(branch, None) check(shard is not None) return await shard.add_block_list_for_sync(block_list)
async def send_minor_block_header_to_master(self, minor_block_header, tx_count, x_shard_tx_count, shard_stats): """ Update master that a minor block has been appended successfully """ request = AddMinorBlockHeaderRequest(minor_block_header, tx_count, x_shard_tx_count, shard_stats) _, resp, _ = await self.master.write_rpc_request( ClusterOp.ADD_MINOR_BLOCK_HEADER_REQUEST, request) check(resp.error_code == 0) self.artificial_tx_config = resp.artificial_tx_config
async def add_block_list_for_sync(self, block_list): """ Add blocks in batch to reduce RPCs. Will NOT broadcast to peers. Returns true if blocks are successfully added. False on any error. Additionally, returns list of coinbase_amount_map for each block This function only adds blocks to local and propagate xshard list to other shards. It does NOT notify master because the master should already have the minor header list, and will add them once this function returns successfully. """ coinbase_amount_list = [] if not block_list: return True, coinbase_amount_list existing_add_block_futures = [] block_hash_to_x_shard_list = dict() for block in block_list: check( block.header.branch.get_full_shard_id() == self.full_shard_id) block_hash = block.header.get_hash() try: xshard_list, coinbase_amount_map = self.state.add_block( block, skip_if_too_old=False) # coinbase_amount_map may be None if the block exists # adding the block header one since the block is already validated. coinbase_amount_list.append(block.header.coinbase_amount_map) except Exception as e: Logger.error_exception() return False, coinbase_amount_list # block already existed in local shard state # but might not have been propagated to other shards and master # let's make sure all the shards and master got it before return if xshard_list is None: future = self.add_block_futures.get(block_hash, None) if future: existing_add_block_futures.append(future) else: prev_root_height = self.state.db.get_root_block_header_by_hash( block.header.hash_prev_root_block).height block_hash_to_x_shard_list[block_hash] = (xshard_list, prev_root_height) self.add_block_futures[block_hash] = self.loop.create_future() await self.slave.batch_broadcast_xshard_tx_list( block_hash_to_x_shard_list, block_list[0].header.branch) for block_hash in block_hash_to_x_shard_list.keys(): self.add_block_futures[block_hash].set_result(None) del self.add_block_futures[block_hash] await asyncio.gather(*existing_add_block_futures) return True, coinbase_amount_list
def mine_next_minor_block(self, ts): check(self.minor_block_to_mine is None) m_block = self.get_next_minor_block_to_mine(ts) if m_block is None: return self.minor_block_to_mine = m_block self.scheduler.schedule_after(m_block.mined_time - ts, self.mine_minor_block, self.minor_block_to_mine)
async def get_work(self, branch: Branch) -> Optional[MiningWork]: try: shard = self.shards[branch] work, block = await shard.miner.get_work() if shard.state.shard_config.POSW_CONFIG.ENABLED: check(isinstance(block, MinorBlock)) diff = shard.state.posw_diff_adjust(block) work = MiningWork(work.hash, work.height, diff) return work except Exception: Logger.log_exception() return None
def get_block_count(self, root_height, shard_size): """Returns a list(dict(miner_recipient, block_count)) of size shard_size""" shard_recipient_cnt = [dict() for _ in range(shard_size)] if not self.count_minor_blocks: return shard_recipient_cnt for shard in range(shard_size): data = self.db.get(b"count_%d_%d" % (shard, root_height)) check(len(data) % 24 == 0) for i in range(0, len(data), 24): recipient = data[i:i + 20].hex() count = int.from_bytes(data[i + 20:i + 24], "big") shard_recipient_cnt[shard][recipient] = count return shard_recipient_cnt
def is_neighbor(b1: Branch, b2: Branch): """A naive algorithm to decide neighbor relationship Two shards are neighbor iff there is only 1 bit difference in their shard ids. This only applies if there are more than 32 shards in the network. Otherwise all shards are neighbor to each other. TODO: a better algorithm """ check(b1.get_shard_size() == b2.get_shard_size()) check(b1.get_shard_id() != b2.get_shard_id()) if b1.get_shard_size() <= 32: return True return is_p2(abs(b1.get_shard_id() - b2.get_shard_id()))
def get_posw_info( config: POSWConfig, header: Header, stakes: int, block_cnt: Dict[bytes, int], stake_per_block: Optional[int] = None, signer: Optional[bytes] = None, ) -> Optional[PoSWInfo]: if (not (config.ENABLED and header.create_time >= config.ENABLE_TIMESTAMP) or header.height == 0): return None # evaluate stakes before the to-be-added block coinbase_recipient = header.coinbase_address.recipient required_stakes_per_block = stake_per_block or config.TOTAL_STAKE_PER_BLOCK block_threshold = min(config.WINDOW_SIZE, stakes // required_stakes_per_block) cnt = block_cnt.get(coinbase_recipient, 0) diff = header.difficulty ret = lambda success: PoSWInfo( diff // config.get_diff_divider(header.create_time) if success else diff, block_threshold, # mined blocks should include current one, assuming success posw_mined_blocks=cnt + 1, ) # fast path if block_threshold == 0: return ret(False) # need to check signature if signer is specified. only applies for root chain if signer: check(isinstance(header, RootBlockHeader)) if signer == bytes(20): return ret(False) block_sig = Signature(header.signature) try: pubk = block_sig.recover_public_key_from_msg_hash( header.get_hash_for_mining()) except BadSignature: return ret(False) if pubk.to_canonical_address() != signer: return ret(False) return ret(cnt < block_threshold)
def get_test_env( genesis_account=Address.create_empty_account(), genesis_minor_quarkash=0, shard_size=2, genesis_root_heights=None, remote_mining=False, ): env = DEFAULT_ENV.copy() env.db = InMemoryDb() env.set_network_id(1234567890) env.cluster_config = ClusterConfig() env.quark_chain_config.update(shard_size, 10, 1) if remote_mining: env.quark_chain_config.ROOT.CONSENSUS_CONFIG.REMOTE_MINE = True env.quark_chain_config.ROOT.CONSENSUS_TYPE = ConsensusType.POW_SHA3SHA3 env.quark_chain_config.ROOT.GENESIS.DIFFICULTY = 10 env.quark_chain_config.ROOT.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME = 40 env.quark_chain_config.ROOT.DIFFICULTY_ADJUSTMENT_FACTOR = 1024 if genesis_root_heights: check(len(genesis_root_heights) == shard_size) for shard_id in range(shard_size): shard = env.quark_chain_config.SHARD_LIST[shard_id] shard.GENESIS.ROOT_HEIGHT = genesis_root_heights[shard_id] # fund genesis account in all shards for i, shard in enumerate(env.quark_chain_config.SHARD_LIST): addr = genesis_account.address_in_shard(i).serialize().hex() shard.GENESIS.ALLOC[addr] = genesis_minor_quarkash shard.CONSENSUS_CONFIG.REMOTE_MINE = remote_mining shard.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME = 7 shard.DIFFICULTY_ADJUSTMENT_FACTOR = 512 if remote_mining: shard.CONSENSUS_TYPE = ConsensusType.POW_SHA3SHA3 shard.GENESIS.DIFFICULTY = 10 env.quark_chain_config.SKIP_MINOR_DIFFICULTY_CHECK = True env.quark_chain_config.SKIP_ROOT_DIFFICULTY_CHECK = True env.cluster_config.ENABLE_TRANSACTION_HISTORY = True env.cluster_config.DB_PATH_ROOT = "" check(env.cluster_config.use_mem_db()) return env