def test_mine_ethash_new_block_overwrite(self): # set a super low `rounds`, and put blocks into input queue beforehand # which will make miner consistently drop current block and start mining new one block = RootBlock( RootBlockHeader( create_time=42, # so we have deterministic hash extra_data="{}".encode("utf-8"), difficulty= 5, # low probability on successful mining at first try )) async def create(): nonlocal block return block async def add(block_to_add): nonlocal miner self.added_blocks.append(block_to_add) miner.input_q.put((None, {})) miner = self.miner_gen(ConsensusType.POW_ETHASH, create, add) # only one round! miner.get_mining_param_func = functools.partial(self.get_mining_params, rounds=1) # insert 5 blocks beforehand for _ in range(5): miner.input_q.put((block, {})) loop = asyncio.get_event_loop() loop.run_until_complete(miner._mine_new_block_async()) # will only have 1 block mined self.assertEqual(len(self.added_blocks), 1)
async def create(): nonlocal i if i >= 5: return None return RootBlock( RootBlockHeader(create_time=int(time.time()), extra_data="{}".encode("utf-8")))
async def create(retry=True): if len(self.added_blocks) >= 5: return None # stop the game return RootBlock( RootBlockHeader(create_time=int(time.time())), tracking_data="{}".encode("utf-8"), )
def test_submit_work_with_guardian(self): now = 42 block = RootBlock( RootBlockHeader(create_time=42, extra_data=b"{}", difficulty=1000)) async def create(retry=True): return block async def add(_): pass miner = self.miner_gen( ConsensusType.POW_DOUBLESHA256, create, add, remote=True, # fake pk, will succeed in test but fail in real world when # adding the block to the root chain guardian_private_key=ecies.generate_privkey(), ) async def go(): for i in range(42, 100): work = await miner.get_work(now=now) self.assertEqual(work.height, 0) # guardian: diff 1000 -> 1, any number should work res = await miner.submit_work(work.hash, i, sha3_256(b"")) self.assertTrue(res) loop = asyncio.get_event_loop() loop.run_until_complete(go())
async def create(retry=True): nonlocal i if i >= 5: return None return RootBlock( RootBlockHeader(create_time=int(time.time())), tracking_data="{}".encode("utf-8"), )
def get_root_block_by_hash(self, h, consistency_check=True): if consistency_check and h not in self.r_header_pool: return None raw_block = self.db.get(b"rblock_" + h, None) if not raw_block: return None return RootBlock.deserialize(raw_block)
async def send_ping(self): # TODO: Send real root tip and allow shards to confirm each other req = Ping( self.slave_server.id, self.slave_server.shard_mask_list, RootBlock(RootBlockHeader()), ) op, resp, rpc_id = await self.write_rpc_request(ClusterOp.PING, req) return (resp.id, resp.shard_mask_list)
def get_root_block_by_hash(self, h) -> Optional[RootBlock]: key = b"rblock_" + h if key in self.rblock_cache: return self.rblock_cache[key] raw_block = self.db.get(key, None) block = raw_block and RootBlock.deserialize(raw_block) if block is not None: self.rblock_cache[key] = block return block
def __recover_from_db(self): """ Recover the best chain from local database. """ Logger.info("Recovering root chain from local database...") if b"tipHash" not in self.db: return None r_hash = self.db.get(b"tipHash") r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash)) if r_block.header.height <= 0: return None # use the parent of the tipHash block as the new tip # since it's guaranteed to have been accepted by all the shards # while shards might not have seen the block of tipHash r_hash = r_block.header.hash_prev_block r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash)) self.tip_header = r_block.header # type: RootBlockHeader
async def create(**kwargs): nonlocal now, mock_tip return RootBlock( RootBlockHeader( create_time=now, extra_data=b"{}", hash_prev_block=mock_tip.get_hash(), height=mock_tip.height + 1, ))
def test_submit_work(self): now, height = 42, 42 doublesha = ConsensusType.POW_DOUBLESHA256 mock_tip = RootBlockHeader(height=height) block = RootBlock( RootBlockHeader(create_time=42, extra_data=b"{}", difficulty=5)) async def create(coinbase_addr=None, retry=True): nonlocal block, mock_tip ret = copy.deepcopy(block) ret.header.height = mock_tip.height + 1 ret.header.hash_prev_block = mock_tip.get_hash() return ret async def add(block_to_add): validate_seal(block_to_add.header, doublesha) self.added_blocks.append(block_to_add) def tip_getter(): nonlocal mock_tip return mock_tip miner = self.miner_gen(doublesha, create, add, tip_getter, remote=True) async def go(): nonlocal mock_tip work, _ = await miner.get_work(EMPTY_ADDR, now=now) self.assertEqual(work.height, 43) self.assertEqual(work.difficulty, 5) # submitted block doesn't exist res = await miner.submit_work(b"lolwut", 0, sha3_256(b"")) self.assertFalse(res) solver = DoubleSHA256(work) sol = solver.mine(200, 300).nonce self.assertGreater(sol, 200) # ensure non-solution is tried non_sol = sol - 1 # invalid pow proof res = await miner.submit_work(work.hash, non_sol, sha3_256(b"")) self.assertFalse(res) # valid submission, but tip updated so should fail mock_tip.height += 1 res = await miner.submit_work(work.hash, sol, sha3_256(b"")) self.assertFalse(res) self.assertEqual(miner.work_map, {}) # bring tip back and regenerate the work mock_tip.height -= 1 work, _ = await miner.get_work(EMPTY_ADDR, now=now) # valid submission, also check internal state afterwards res = await miner.submit_work(work.hash, sol, sha3_256(b"")) self.assertTrue(res) self.assertEqual(miner.work_map, {}) self.assertEqual(len(self.added_blocks), 1) loop = asyncio.get_event_loop() loop.run_until_complete(go())
def create_root_block(self) -> RootBlock: """ Create the genesis root block """ genesis = self._qkc_config.ROOT.GENESIS header = RootBlockHeader( version=genesis.VERSION, height=genesis.HEIGHT, hash_prev_block=bytes.fromhex(genesis.HASH_PREV_BLOCK), hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT), create_time=genesis.TIMESTAMP, difficulty=genesis.DIFFICULTY, ) return RootBlock(header=header, minor_block_header_list=[])
def test_sha3sha3(self): miner = self.miner_gen(ConsensusType.POW_SHA3SHA3, None, None) block = RootBlock( RootBlockHeader(create_time=42, extra_data="{}".encode("utf-8"), difficulty=5)) # only process one block, which is passed in miner.input_q.put((None, {})) miner.mine_sha3sha3(block, miner.input_q, miner.output_q, {}) mined_block = miner.output_q.get() self.assertEqual(mined_block.header.nonce, 3) validate_seal(mined_block.header, ConsensusType.POW_SHA3SHA3)
def get_root_block_by_hash(self, h, consistency_check=True): """ Consistency check being true means whatever in memory should already have enough information (no missing root block, minor block etc). Skipping the check by reading directly from database may have unwanted consequences. """ if consistency_check and h not in self.r_header_pool: return None raw_block = self.db.get(b"rblock_" + h, None) if not raw_block: return None return RootBlock.deserialize(raw_block)
def __recover_from_db(self): """ Recover the best chain from local database. """ Logger.info("Recovering root chain from local database...") if b"tipHash" not in self.db: return None r_hash = self.db.get(b"tipHash") r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash)) self.tip_header = r_block.header while len(self.r_header_pool) < self.max_num_blocks_to_recover: self.r_header_pool[r_hash] = r_block.header for m_header in r_block.minor_block_header_list: self.m_hash_set.add(m_header.get_hash()) if r_block.header.height <= 0: break r_hash = r_block.header.hash_prev_block r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash))
def test_submit_work_with_remote_guardian(self): now = 42 doublesha = ConsensusType.POW_DOUBLESHA256 block = RootBlock( RootBlockHeader( create_time=42, extra_data=b"{}", difficulty=1000, hash_prev_block=RootBlockHeader().get_hash(), )) priv = ecies.generate_privkey() async def create(coinbase_addr=None, retry=True): return block async def add(block_to_add): h = block_to_add.header diff = h.difficulty if h.verify_signature(priv.public_key): diff = Guardian.adjust_difficulty(diff, h.height) validate_seal(block_to_add.header, doublesha, adjusted_diff=diff) # just with the guardian public key miner = self.miner_gen(doublesha, create, add, remote=True) async def go(): for i in range(42, 100): work, _ = await miner.get_work(EMPTY_ADDR, now=now) self.assertEqual(work.height, 0) # remote guardian: diff 1000 -> 1, any number should work # mimic the sign process of the remote guardian server block.header.nonce = i block.header.mixhash = sha3_256(b"") block.header.sign_with_private_key(priv) signature = block.header.signature # reset the signature to the default value block.header.signature = bytes(65) # submit the signature through the submit work res = await miner.submit_work(work.hash, i, sha3_256(b""), signature) self.assertTrue(res) res = await miner.submit_work(work.hash, i, sha3_256(b""), bytes(65)) self.assertFalse(res) loop = asyncio.get_event_loop() loop.run_until_complete(go())
def test_validate_seal_with_adjusted_diff(self): diff = 1000 block = RootBlock( RootBlockHeader(create_time=42, difficulty=diff), tracking_data="{}".encode("utf-8"), ) block.header.nonce = 0 with self.assertRaises(ValueError): validate_seal(block.header, ConsensusType.POW_DOUBLESHA256) # significantly lowering the diff should pass validate_seal(block.header, ConsensusType.POW_DOUBLESHA256, adjusted_diff=1)
def test_sha3sha3(self): miner = self.miner_gen(ConsensusType.POW_SHA3SHA3, None, None) block = RootBlock( RootBlockHeader(create_time=42, difficulty=5), tracking_data="{}".encode("utf-8"), ) work = MiningWork(block.header.get_hash_for_mining(), 42, 5) # only process one block, which is passed in. `None` means termination right after miner.input_q.put((None, {})) miner._mine_loop( ConsensusType.POW_SHA3SHA3, work, {}, miner.input_q, miner.output_q ) mined_res = miner.output_q.get() self.assertEqual(mined_res.nonce, 8) block.header.nonce = mined_res.nonce validate_seal(block.header, ConsensusType.POW_SHA3SHA3)
def test_qkchash(self): miner = self.miner_gen(ConsensusType.POW_QKCHASH, None, None) block = RootBlock( RootBlockHeader(create_time=42, difficulty=5), tracking_data="{}".encode("utf-8"), ) work = MiningWork(block.header.get_hash_for_mining(), 42, 5) # only process one block, which is passed in. `None` means termination right after miner.input_q.put((None, {})) miner.mine_loop( work, {"consensus_type": ConsensusType.POW_QKCHASH}, miner.input_q, miner.output_q, ) mined_res = miner.output_q.get() block.header.nonce = mined_res.nonce block.header.mixhash = mined_res.mixhash validate_seal(block.header, ConsensusType.POW_QKCHASH)
def test_submit_work(self): now = 42 block = RootBlock( RootBlockHeader(create_time=42, extra_data=b"{}", difficulty=2)) async def create(): return block async def add(block_to_add): self.added_blocks.append(block_to_add) miner = self.miner_gen(ConsensusType.POW_SHA3SHA3, create, add, remote=True) async def go(): work = await miner.get_work(now=now) self.assertEqual(work.height, 0) self.assertEqual(work.difficulty, 2) # submitted block doesn't exist res = await miner.submit_work(b"lolwut", 0, sha3_256(b"")) self.assertFalse(res) solver = DoubleSHA256(block) mined = solver.mine(100, 200) self.assertTrue(mined) sol = int.from_bytes(solver.nonce_found, byteorder="big") self.assertGreater(sol, 100) # ensure non-solution is tried non_sol = sol - 1 # invalid pow proof res = await miner.submit_work(work.hash, non_sol, sha3_256(b"")) self.assertFalse(res) # valid submission, also check internal state afterwards res = await miner.submit_work(work.hash, sol, sha3_256(b"")) self.assertTrue(res) self.assertEqual(miner.work_map, {}) self.assertEqual(len(self.added_blocks), 1) self.assertIsNone(miner.current_work) loop = asyncio.get_event_loop() loop.run_until_complete(go())
def test_submit_work(self): now = 42 doublesha = ConsensusType.POW_DOUBLESHA256 block = RootBlock( RootBlockHeader(create_time=42, extra_data=b"{}", difficulty=5)) async def create(retry=True): return block async def add(block_to_add): validate_seal(block_to_add.header, doublesha) self.added_blocks.append(block_to_add) miner = self.miner_gen(doublesha, create, add, remote=True) async def go(): work, _ = await miner.get_work(now=now) self.assertEqual(work.height, 0) self.assertEqual(work.difficulty, 5) # submitted block doesn't exist res = await miner.submit_work(b"lolwut", 0, sha3_256(b"")) self.assertFalse(res) solver = DoubleSHA256(work) sol = solver.mine(200, 300).nonce self.assertGreater(sol, 200) # ensure non-solution is tried non_sol = sol - 1 # invalid pow proof res = await miner.submit_work(work.hash, non_sol, sha3_256(b"")) self.assertFalse(res) # valid submission, also check internal state afterwards res = await miner.submit_work(work.hash, sol, sha3_256(b"")) self.assertTrue(res) self.assertEqual(miner.work_map, {}) self.assertEqual(len(self.added_blocks), 1) self.assertIsNone(miner.current_work) loop = asyncio.get_event_loop() loop.run_until_complete(go())
def test_submit_work_with_guardian(self): now = 42 doublesha = ConsensusType.POW_DOUBLESHA256 block = RootBlock( RootBlockHeader( create_time=42, extra_data=b"{}", difficulty=1000, hash_prev_block=RootBlockHeader().get_hash(), )) priv = ecies.generate_privkey() async def create(coinbase_addr=None, retry=True): return block async def add(block_to_add): h = block_to_add.header diff = h.difficulty if h.verify_signature(priv.public_key): diff = Guardian.adjust_difficulty(diff, h.height) validate_seal(block_to_add.header, doublesha, adjusted_diff=diff) miner = self.miner_gen(doublesha, create, add, remote=True, guardian_private_key=priv) async def go(): for i in range(42, 100): work, _ = await miner.get_work(EMPTY_ADDR, now=now) self.assertEqual(work.height, 0) # guardian: diff 1000 -> 1, any number should work res = await miner.submit_work(work.hash, i, sha3_256(b"")) self.assertTrue(res) loop = asyncio.get_event_loop() loop.run_until_complete(go())
async def addBlock(self, branch, block_data): if branch == 0: block = RootBlock.deserialize(block_data) return await self.master.add_root_block_from_miner(block) return await self.master.add_raw_minor_block(Branch(branch), block_data)
async def dummy_create_block_async() -> Optional[RootBlock]: if len(TestMiner.added_blocks) >= 5: return None # stop the game return RootBlock(RootBlockHeader(create_time=int(time.time()), extra_data="{}".encode("utf-8")))
def test_getNextBlockToMine_and_addBlock(self): id1 = Identity.create_random_identity() acc1 = Address.create_from_identity(id1, full_shard_id=0) acc3 = Address.create_random_account(full_shard_id=1) with ClusterContext(1, acc1) as clusters, jrpc_server_context( clusters[0].master): slaves = clusters[0].slave_list # Expect to mine root that confirms the genesis minor blocks response = send_request("getNextBlockToMine", "0x" + acc1.serialize().hex(), "0x0") self.assertTrue(response["isRootBlock"]) block = RootBlock.deserialize( bytes.fromhex(response["blockData"][2:])) self.assertEqual(block.header.height, 1) self.assertEqual(len(block.minor_block_header_list), 2) self.assertEqual(block.minor_block_header_list[0].height, 0) self.assertEqual(block.minor_block_header_list[1].height, 0) send_request("addBlock", "0x0", response["blockData"]) tx = create_transfer_transaction( shard_state=clusters[0].get_shard_state(0), key=id1.get_key(), from_address=acc1, to_address=acc3, value=14, gas=opcodes.GTXXSHARDCOST + opcodes.GTXCOST, ) self.assertTrue(slaves[0].add_tx(tx)) # Expect to mine shard 0 since it has one tx response = send_request("getNextBlockToMine", "0x" + acc1.serialize().hex(), "0x0") self.assertFalse(response["isRootBlock"]) block1 = MinorBlock.deserialize( bytes.fromhex(response["blockData"][2:])) self.assertEqual(block1.header.branch.value, 0b10) self.assertTrue( send_request("addBlock", "0x2", response["blockData"])) self.assertEqual( clusters[0].get_shard_state(1).get_balance(acc3.recipient, 0), 0) # Expect to mine shard 1 due to proof-of-progress response = send_request("getNextBlockToMine", "0x" + acc1.serialize().hex(), "0x0") self.assertFalse(response["isRootBlock"]) block2 = MinorBlock.deserialize( bytes.fromhex(response["blockData"][2:])) self.assertEqual(block2.header.branch.value, 0b11) self.assertTrue( send_request("addBlock", "0x3", response["blockData"])) # Expect to mine root response = send_request("getNextBlockToMine", "0x" + acc1.serialize().hex(), "0x0") self.assertTrue(response["isRootBlock"]) block = RootBlock.deserialize( bytes.fromhex(response["blockData"][2:])) self.assertEqual(block.header.height, 2) self.assertEqual(len(block.minor_block_header_list), 2) self.assertEqual(block.minor_block_header_list[0], block1.header) self.assertEqual(block.minor_block_header_list[1], block2.header) send_request("addBlock", "0x0", response["blockData"]) self.assertEqual( clusters[0].get_shard_state(1).get_balance(acc3.recipient, 0), 0) # Expect to mine shard 1 for the gas on xshard tx to acc3 response = send_request("getNextBlockToMine", "0x" + acc1.serialize().hex(), "0x0") self.assertFalse(response["isRootBlock"]) block3 = MinorBlock.deserialize( bytes.fromhex(response["blockData"][2:])) self.assertEqual(block3.header.branch.value, 0b11) self.assertTrue( send_request("addBlock", "0x3", response["blockData"])) # Expect withdrawTo is included in acc3's balance resp = send_request("getBalance", "0x" + acc3.serialize().hex()) self.assertEqual(resp["branch"], "0x3") self.assertEqual(resp["balance"], "0xe")
async def create(retry=True): nonlocal now return RootBlock(RootBlockHeader(create_time=now, extra_data=b"{}"))
def get_root_block_by_hash(self, h): if h not in self.r_header_pool: return None return RootBlock.deserialize(self.db.get(b"rblock_" + h))