def from_dict(cls, d): config = super().from_dict(d) config.ROOT = RootConfig.from_dict(config.ROOT) chains = [] shards = dict() for s in config.CHAINS: chain_config = ChainConfig.from_dict(s) chains.append(chain_config) for shard_id in range(chain_config.SHARD_SIZE): shard_config = ShardConfig(chain_config) shard_config.root_config = config.ROOT shard_config.SHARD_ID = shard_id shard_config.COINBASE_ADDRESS = (Address.create_from( shard_config.COINBASE_ADDRESS).address_in_shard( shard_config.get_full_shard_id()).serialize().hex()) # filter alloc addresses based on shard id alloc = dict() for address_hex, balances in shard_config.GENESIS.ALLOC.items( ): address = Address.create_from(bytes.fromhex(address_hex)) address_shard_id = address.full_shard_key & ( chain_config.SHARD_SIZE - 1) if shard_id == address_shard_id: alloc[address_hex] = balances shard_config.GENESIS.ALLOC = alloc shards[shard_config.get_full_shard_id()] = shard_config config.CHAINS = chains config.shards = shards config.init_and_validate() return config
def update_genesis_alloc(cluser_config): """ Update ShardConfig.GENESIS.ALLOC """ ALLOC_FILE_TEMPLATE = "alloc/{}.json" LOADTEST_FILE = "loadtest.json" if not cluser_config.GENESIS_DIR: return alloc_file_template = os.path.join(cluser_config.GENESIS_DIR, ALLOC_FILE_TEMPLATE) loadtest_file = os.path.join(cluser_config.GENESIS_DIR, LOADTEST_FILE) qkc_config = cluser_config.QUARKCHAIN allocation = { qkc_config.GENESIS_TOKEN: 1000000 * (10**18), "QETC": 2 * (10**8) * (10**18), "QFB": 3 * (10**8) * (10**18), "QAAPL": 4 * (10**8) * (10**18), "QTSLA": 5 * (10**8) * (10**18), } old_shards = copy.deepcopy(qkc_config.shards) try: for chain_id in range(qkc_config.CHAIN_SIZE): alloc_file = alloc_file_template.format(chain_id) with open(alloc_file, "r") as f: items = json.load(f) for item in items: address = Address.create_from(item["address"]) full_shard_id = qkc_config.get_full_shard_id_by_full_shard_key( address.full_shard_key) qkc_config.shards[full_shard_id].GENESIS.ALLOC[ item["address"]] = allocation Logger.info( "[{}] Imported {} genesis accounts into config from {}".format( chain_id, len(items), alloc_file)) except Exception as e: Logger.warning("Error importing genesis accounts from {}: {}".format( alloc_file, e)) qkc_config.shards = old_shards Logger.warning("Cleared all partially imported genesis accounts!") # each account in loadtest file is funded on all the shards try: with open(loadtest_file, "r") as f: items = json.load(f) qkc_config.loadtest_accounts = items for item in items: address = Address.create_from(item["address"]) for full_shard_id, shard_config in qkc_config.shards.items(): shard_config.GENESIS.ALLOC[address.address_in_shard( full_shard_id).serialize().hex()] = allocation Logger.info("Imported {} loadtest accounts from {}".format( len(items), loadtest_file)) except Exception: Logger.info("No loadtest accounts imported into genesis alloc")
def test_getWork_and_submitWork(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_key=0) with ClusterContext( 1, acc1, remote_mining=True, shard_size=1, small_coinbase=True) as clusters, jrpc_server_context( clusters[0].master): master = clusters[0].master slaves = clusters[0].slave_list tx = create_transfer_transaction( shard_state=clusters[0].get_shard_state(1 | 0), key=id1.get_key(), from_address=acc1, to_address=acc1, value=0, gas_price=12, ) self.assertTrue(slaves[0].add_tx(tx)) for shard_id in ["0x0", None]: # shard, then root resp = send_request("getWork", [shard_id]) self.assertEqual(resp[1:], ["0x1", "0xa"]) # height and diff header_hash_hex = resp[0] if shard_id is not None: # shard 0 miner_address = Address.create_from( master.env.quark_chain_config.shards[1]. COINBASE_ADDRESS) else: # root miner_address = Address.create_from( master.env.quark_chain_config.ROOT.COINBASE_ADDRESS) block = call_async( master.get_next_block_to_mine(address=miner_address, branch_value=shard_id and 0b01)) # solve it and submit work = MiningWork(bytes.fromhex(header_hash_hex[2:]), 1, 10) solver = DoubleSHA256(work) nonce = solver.mine(0, 10000).nonce mixhash = "0x" + sha3_256(b"").hex() resp = send_request( "submitWork", [ shard_id, header_hash_hex, hex(nonce), mixhash, "0x" + bytes(65).hex(), ], ) self.assertTrue(resp) # show progress on shard 0 self.assertEqual( clusters[0].get_shard_state(1 | 0).get_tip().header.height, 1)
def update_genesis_alloc(cluser_config): """ Update ShardConfig.GENESIS.ALLOC """ ALLOC_FILE = "alloc.json" LOADTEST_FILE = "loadtest.json" if not cluser_config.GENESIS_DIR: return alloc_file = os.path.join(cluser_config.GENESIS_DIR, ALLOC_FILE) loadtest_file = os.path.join(cluser_config.GENESIS_DIR, LOADTEST_FILE) qkc_config = cluser_config.QUARKCHAIN # each account in alloc_file is only funded on the shard it belongs to try: with open(alloc_file, "r") as f: items = json.load(f) # address_hex -> key_hex for jsonrpc faucet drip qkc_config.alloc_accounts = dict() for item in items: qkc_config.alloc_accounts[item["address"]] = item["key"] address = Address.create_from(item["address"]) shard = address.get_shard_id(qkc_config.SHARD_SIZE) qkc_config.SHARD_LIST[shard].GENESIS.ALLOC[item["address"]] = 1000000 * ( 10 ** 18 ) Logger.info( "Imported {} accounts from genesis alloc at {}".format( len(items), alloc_file ) ) except Exception as e: Logger.warning("Unable to load genesis alloc from {}: {}".format(alloc_file, e)) # each account in loadtest file is funded on all the shards try: with open(loadtest_file, "r") as f: items = json.load(f) qkc_config.loadtest_accounts = items for item in items: address = Address.create_from(item["address"]) for i, shard in enumerate(qkc_config.SHARD_LIST): shard.GENESIS.ALLOC[ address.address_in_shard(i).serialize().hex() ] = 1000 * (10 ** 18) Logger.info( "Imported {} loadtest accounts from {}".format(len(items), loadtest_file) ) except Exception: Logger.info("No loadtest accounts imported into genesis alloc")
def _convert_eth_call_data(data, shard): to_address = Address.create_from( eth_address_to_quarkchain_address_decoder(data["to"])) if shard: to_address = Address(to_address.recipient, shard) data["to"] = "0x" + to_address.serialize().hex() if "from" in data: from_address = Address.create_from( eth_address_to_quarkchain_address_decoder(data["from"])) if shard: from_address = Address(from_address.recipient, shard) data["from"] = "0x" + from_address.serialize().hex() return data
def update(self, chain_size, shard_size_per_chain, root_block_time, minor_block_time): self.CHAIN_SIZE = chain_size self.ROOT = RootConfig() self.ROOT.CONSENSUS_TYPE = ConsensusType.POW_SIMULATE self.ROOT.CONSENSUS_CONFIG = POWConfig() self.ROOT.CONSENSUS_CONFIG.TARGET_BLOCK_TIME = root_block_time self.CHAINS = [] self.shards = dict() for chain_id in range(chain_size): chain_config = ChainConfig() chain_config.CHAIN_ID = chain_id chain_config.SHARD_SIZE = shard_size_per_chain chain_config.CONSENSUS_TYPE = ConsensusType.POW_SIMULATE chain_config.CONSENSUS_CONFIG = POWConfig() chain_config.CONSENSUS_CONFIG.TARGET_BLOCK_TIME = minor_block_time self.CHAINS.append(chain_config) for shard_id in range(shard_size_per_chain): shard_config = ShardConfig(chain_config) shard_config.root_config = self.ROOT shard_config.SHARD_ID = shard_id shard_config.COINBASE_ADDRESS = (Address.create_from( shard_config.COINBASE_ADDRESS).address_in_shard( shard_config.get_full_shard_id()).serialize().hex()) self.shards[shard_config.get_full_shard_id()] = shard_config self.init_and_validate()
async def getTransactionsByAddress(self, address, start="0x", limit="0xa"): """ "start" should be the "next" in the response for fetching next page. "start" can also be "0x" to fetch from the beginning (i.e., latest). "start" can be "0x00" to fetch the pending outgoing transactions. """ address = Address.create_from(address) if limit > 20: limit = 20 result = await self.master.get_transactions_by_address( address, start, limit) if not result: return None tx_list, next = result txs = [] for tx in tx_list: txs.append({ "txId": id_encoder(tx.tx_hash, tx.from_address.full_shard_id), "fromAddress": address_encoder(tx.from_address.serialize()), "toAddress": address_encoder(tx.to_address.serialize()) if tx.to_address else "0x", "value": quantity_encoder(tx.value), "blockHeight": quantity_encoder(tx.block_height), "timestamp": quantity_encoder(tx.timestamp), "success": tx.success, }) return {"txList": txs, "next": data_encoder(next)}
def create_minor_block(self, root_block: RootBlock, full_shard_id: int, evm_state: EvmState) -> MinorBlock: """ Create genesis block for shard. Genesis block's hash_prev_root_block is set to the genesis root block. Genesis state will be committed to the given evm_state. Based on ALLOC, genesis_token will be added to initial accounts. """ branch = Branch(full_shard_id) shard_config = self._qkc_config.shards[full_shard_id] genesis = shard_config.GENESIS for address_hex, alloc_amount in genesis.ALLOC.items(): address = Address.create_from(bytes.fromhex(address_hex)) check( self._qkc_config.get_full_shard_id_by_full_shard_key( address.full_shard_key) == full_shard_id) evm_state.full_shard_key = address.full_shard_key if isinstance(alloc_amount, dict): for k, v in alloc_amount.items(): evm_state.delta_token_balance(address.recipient, token_id_encode(k), v) else: evm_state.delta_token_balance(address.recipient, self._qkc_config.genesis_token, alloc_amount) evm_state.commit() meta = MinorBlockMeta( hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT), hash_evm_state_root=evm_state.trie.root_hash, ) local_fee_rate = 1 - self._qkc_config.reward_tax_rate # type: Fraction coinbase_tokens = { self._qkc_config.genesis_token: shard_config.COINBASE_AMOUNT * local_fee_rate.numerator // local_fee_rate.denominator } coinbase_address = Address.create_empty_account(full_shard_id) header = MinorBlockHeader( version=genesis.VERSION, height=genesis.HEIGHT, branch=branch, hash_prev_minor_block=bytes.fromhex(genesis.HASH_PREV_MINOR_BLOCK), hash_prev_root_block=root_block.header.get_hash(), evm_gas_limit=genesis.GAS_LIMIT, hash_meta=sha3_256(meta.serialize()), coinbase_amount_map=TokenBalanceMap(coinbase_tokens), coinbase_address=coinbase_address, create_time=genesis.TIMESTAMP, difficulty=genesis.DIFFICULTY, extra_data=bytes.fromhex(genesis.EXTRA_DATA), ) return ( MinorBlock(header=header, meta=meta, tx_list=[]), TokenBalanceMap(coinbase_tokens), )
async def _call_or_estimate_gas(self, is_call: bool, **data): """ Returns the result of the transaction application without putting in block chain """ if not isinstance(data, dict): raise InvalidParams("Transaction must be an object") def get_data_default(key, decoder, default=None): if key in data: return decoder(data[key]) return default to = get_data_default("to", address_decoder, None) if to is None: raise InvalidParams("Missing to") to_full_shard_key = int.from_bytes(to[20:], "big") gas = get_data_default("gas", quantity_decoder, 0) gas_price = get_data_default("gasPrice", quantity_decoder, 0) value = get_data_default("value", quantity_decoder, 0) data_ = get_data_default("data", data_decoder, b"") sender = get_data_default("from", address_decoder, b"\x00" * 20 + to[20:]) sender_address = Address.create_from(sender) gas_token_id = get_data_default( "gas_token_id", quantity_decoder, self.env.quark_chain_config.genesis_token) transfer_token_id = get_data_default( "transfer_token_id", quantity_decoder, self.env.quark_chain_config.genesis_token, ) network_id = self.master.env.quark_chain_config.NETWORK_ID nonce = 0 # slave will fill in the real nonce evm_tx = EvmTransaction( nonce, gas_price, gas, to[:20], value, data_, from_full_shard_key=sender_address.full_shard_key, to_full_shard_key=to_full_shard_key, network_id=network_id, gas_token_id=gas_token_id, transfer_token_id=transfer_token_id, ) tx = TypedTransaction(SerializedEvmTransaction.from_evm_tx(evm_tx)) if is_call: res = await self.master.execute_transaction( tx, sender_address, data["block_height"]) return data_encoder(res) if res is not None else None else: # estimate gas res = await self.master.estimate_gas(tx, sender_address) return quantity_encoder(res) if res is not None else None
def create_minor_block( self, root_block: RootBlock, shard_id: int, evm_state: EvmState ) -> MinorBlock: """ Create genesis block for shard. Genesis block's hash_prev_root_block is set to the genesis root block. Genesis state will be committed to the given evm_state. """ branch = Branch.create(self._qkc_config.SHARD_SIZE, shard_id) shard_config = self._qkc_config.SHARD_LIST[shard_id] genesis = shard_config.GENESIS coinbase_address = Address.create_from( bytes.fromhex(shard_config.COINBASE_ADDRESS) ) check(coinbase_address.get_shard_id(self._qkc_config.SHARD_SIZE) == shard_id) for address_hex, amount_in_wei in genesis.ALLOC.items(): address = Address.create_from(bytes.fromhex(address_hex)) check(address.get_shard_id(self._qkc_config.SHARD_SIZE) == shard_id) evm_state.full_shard_id = address.full_shard_id evm_state.delta_balance(address.recipient, amount_in_wei) evm_state.commit() meta = MinorBlockMeta( hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT), hash_evm_state_root=evm_state.trie.root_hash, ) header = MinorBlockHeader( version=genesis.VERSION, height=genesis.HEIGHT, branch=branch, hash_prev_minor_block=bytes.fromhex(genesis.HASH_PREV_MINOR_BLOCK), hash_prev_root_block=root_block.header.get_hash(), evm_gas_limit=genesis.GAS_LIMIT, hash_meta=sha3_256(meta.serialize()), coinbase_address=coinbase_address, coinbase_amount=shard_config.COINBASE_AMOUNT, create_time=genesis.TIMESTAMP, difficulty=genesis.DIFFICULTY, extra_data=bytes.fromhex(genesis.EXTRA_DATA), ) return MinorBlock(header=header, meta=meta, tx_list=[])
def __init__(self, qkc_config: QuarkChainConfig, shard): self.qkc_config = qkc_config self.full_shard_id = shard.full_shard_id self.shard = shard self.running = False self.accounts = [] for item in qkc_config.loadtest_accounts: account = Account(Address.create_from(item["address"]), bytes.fromhex(item["key"])) self.accounts.append(account)
async def fund(endpoint, genesisId, data): network_id = await endpoint.get_network_id() shard_size = await endpoint.get_shard_size() futures = [] for e in GAME_ADDRESSES: shard = e[0] to = Address.create_from(e[1][2:]) futures.append( fund_shard(endpoint, genesisId, to, data, network_id, shard)) results = await asyncio.gather(*futures) print("\n\n") for shard, result in enumerate(results): tx_id, height = result print('[{}, "{}"], // {}'.format(shard, height, tx_id))
def from_dict(cls, d): config = super().from_dict(d) config.ROOT = RootConfig.from_dict(config.ROOT) chains = [] shards = dict() for s in config.CHAINS: chain_config = ChainConfig.from_dict(s) chains.append(chain_config) for shard_id in range(chain_config.SHARD_SIZE): shard_config = ShardConfig(chain_config) shard_config.root_config = config.ROOT shard_config.SHARD_ID = shard_id shard_config.COINBASE_ADDRESS = (Address.create_from( shard_config.COINBASE_ADDRESS).address_in_shard( shard_config.get_full_shard_id()).serialize().hex()) shards[shard_config.get_full_shard_id()] = shard_config config.CHAINS = chains config.shards = shards config.init_and_validate() return config
async def getTransactionsByAddress( self, address, start="0x", limit="0xa", transfer_token_id=None ): """ "start" should be the "next" in the response for fetching next page. "start" can also be "0x" to fetch from the beginning (i.e., latest). "start" can be "0x00" to fetch the pending outgoing transactions. """ address = Address.create_from(address) if limit > 20: limit = 20 result = await self.master.get_transactions_by_address( address, transfer_token_id, start, limit ) if not result: return None tx_list, next = result return { "txList": [tx_detail_encoder(tx) for tx in tx_list], "next": data_encoder(next), }
async def fund(endpoint, genesisId, addrByAmount): network_id = await endpoint.get_network_id() shard_size = await endpoint.get_shard_size() for amount in addrByAmount: addrs = addrByAmount.get(amount, []) print("======\nstart for amount {} with {} address\n======".format( amount, len(addrs))) # shard -> [addr] byShard = defaultdict(list) for addr in addrs: shard = int(addr[-8:], 16) & (shard_size - 1) byShard[shard].append(addr) while True: toFund = [] for addrs in byShard.values(): if addrs: toFund.append(addrs.pop()) if not toFund: break futures = [] for addr in toFund: shard = int(addr[-8:], 16) & (shard_size - 1) try: # sorry but this is user input to = Address.create_from(addr[2:]) except: print("addr format invalid {}".format(addr)) continue await asyncio.sleep(0.1) # slight delay for each call futures.append( fund_shard(endpoint, genesisId, to, network_id, shard, amount)) results = await asyncio.gather(*futures) print("\n\n") for idx, result in enumerate(results): tx_id, height = result print('[{}, "{}"], // {}'.format(idx, height, tx_id))
def __init_miner(self): miner_address = Address.create_from( self.env.quark_chain_config.SHARD_LIST[ self.shard_id].COINBASE_ADDRESS) async def __create_block(retry=True): # hold off mining if the shard is syncing while self.synchronizer.running or not self.state.initialized: if not retry: break await asyncio.sleep(0.1) return self.state.create_block_to_mine(address=miner_address) async def __add_block(block): # Do not add block if there is a sync in progress if self.synchronizer.running: return # Do not add stale block if self.state.header_tip.height >= block.header.height: return await self.handle_new_block(block) def __get_mining_param(): return { "target_block_time": self.slave.artificial_tx_config.target_minor_block_time } shard_config = self.env.quark_chain_config.SHARD_LIST[ self.shard_id] # type: ShardConfig self.miner = Miner( shard_config.CONSENSUS_TYPE, __create_block, __add_block, __get_mining_param, remote=shard_config.CONSENSUS_CONFIG.REMOTE_MINE, )
def testnet_master_address(self): return Address.create_from(self.TESTNET_MASTER_ADDRESS)
def create_minor_block( self, root_block: RootBlock, full_shard_id: int, evm_state: EvmState) -> Tuple[MinorBlock, TokenBalanceMap]: """ Create genesis block for shard. Genesis block's hash_prev_root_block is set to the genesis root block. Genesis state will be committed to the given evm_state. Based on ALLOC, genesis_token will be added to initial accounts. """ branch = Branch(full_shard_id) shard_config = self._qkc_config.shards[full_shard_id] genesis = shard_config.GENESIS for address_hex, alloc_data in genesis.ALLOC.items(): address = Address.create_from(bytes.fromhex(address_hex)) check( self._qkc_config.get_full_shard_id_by_full_shard_key( address.full_shard_key) == full_shard_id) evm_state.full_shard_key = address.full_shard_key recipient = address.recipient if "code" in alloc_data: code = decode_hex(remove_0x_head(alloc_data["code"])) evm_state.set_code(recipient, code) evm_state.set_nonce(recipient, 1) if "storage" in alloc_data: for k, v in alloc_data["storage"].items(): evm_state.set_storage_data( recipient, big_endian_to_int(decode_hex(k[2:])), big_endian_to_int(decode_hex(v[2:])), ) # backward compatible: # v1: {addr: {QKC: 1234}} # v2: {addr: {balances: {QKC: 1234}, code: 0x, storage: {0x12: 0x34}}} balances = alloc_data if "balances" in alloc_data: balances = alloc_data["balances"] for k, v in balances.items(): if k in ("code", "storage"): continue evm_state.delta_token_balance(recipient, token_id_encode(k), v) evm_state.commit() meta = MinorBlockMeta( hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT), hash_evm_state_root=evm_state.trie.root_hash, xshard_tx_cursor_info=XshardTxCursorInfo(root_block.header.height, 0, 0), ) local_fee_rate = 1 - self._qkc_config.reward_tax_rate # type: Fraction coinbase_tokens = { self._qkc_config.genesis_token: shard_config.COINBASE_AMOUNT * local_fee_rate.numerator // local_fee_rate.denominator } coinbase_address = Address.create_empty_account(full_shard_id) header = MinorBlockHeader( version=genesis.VERSION, height=genesis.HEIGHT, branch=branch, hash_prev_minor_block=bytes.fromhex(genesis.HASH_PREV_MINOR_BLOCK), hash_prev_root_block=root_block.header.get_hash(), evm_gas_limit=genesis.GAS_LIMIT, hash_meta=sha3_256(meta.serialize()), coinbase_amount_map=TokenBalanceMap(coinbase_tokens), coinbase_address=coinbase_address, create_time=genesis.TIMESTAMP, difficulty=genesis.DIFFICULTY, extra_data=bytes.fromhex(genesis.EXTRA_DATA), ) return ( MinorBlock(header=header, meta=meta, tx_list=[]), TokenBalanceMap(coinbase_tokens), )
def miner_address(self): return Address.create_from(self.MINER_ADDRESS)