def test_mine_ethash_new_block_overwrite(self):
# set a super low `rounds`, and put blocks into input queue beforehand
# which will make miner consistently drop current block and start mining new one
block = RootBlock(
RootBlockHeader(
create_time=42, # so we have deterministic hash
extra_data="{}".encode("utf-8"),
difficulty=
5, # low probability on successful mining at first try
))
async def create():
nonlocal block
return block
async def add(block_to_add):
nonlocal miner
self.added_blocks.append(block_to_add)
miner.input_q.put((None, {}))
miner = self.miner_gen(ConsensusType.POW_ETHASH, create, add)
# only one round!
miner.get_mining_param_func = functools.partial(self.get_mining_params,
rounds=1)
# insert 5 blocks beforehand
for _ in range(5):
miner.input_q.put((block, {}))
loop = asyncio.get_event_loop()
loop.run_until_complete(miner._mine_new_block_async())
# will only have 1 block mined
self.assertEqual(len(self.added_blocks), 1)
async def create():
nonlocal i
if i >= 5:
return None
return RootBlock(
RootBlockHeader(create_time=int(time.time()),
extra_data="{}".encode("utf-8")))
async def create(retry=True):
if len(self.added_blocks) >= 5:
return None # stop the game
return RootBlock(
RootBlockHeader(create_time=int(time.time())),
tracking_data="{}".encode("utf-8"),
)
def test_submit_work_with_guardian(self):
now = 42
block = RootBlock(
RootBlockHeader(create_time=42, extra_data=b"{}", difficulty=1000))
async def create(retry=True):
return block
async def add(_):
pass
miner = self.miner_gen(
ConsensusType.POW_DOUBLESHA256,
create,
add,
remote=True,
# fake pk, will succeed in test but fail in real world when
# adding the block to the root chain
guardian_private_key=ecies.generate_privkey(),
)
async def go():
for i in range(42, 100):
work = await miner.get_work(now=now)
self.assertEqual(work.height, 0)
# guardian: diff 1000 -> 1, any number should work
res = await miner.submit_work(work.hash, i, sha3_256(b""))
self.assertTrue(res)
loop = asyncio.get_event_loop()
loop.run_until_complete(go())
async def create(retry=True):
nonlocal i
if i >= 5:
return None
return RootBlock(
RootBlockHeader(create_time=int(time.time())),
tracking_data="{}".encode("utf-8"),
)
def get_root_block_by_hash(self, h, consistency_check=True):
if consistency_check and h not in self.r_header_pool:
return None
raw_block = self.db.get(b"rblock_" + h, None)
if not raw_block:
return None
return RootBlock.deserialize(raw_block)
async def send_ping(self):
# TODO: Send real root tip and allow shards to confirm each other
req = Ping(
self.slave_server.id,
self.slave_server.shard_mask_list,
RootBlock(RootBlockHeader()),
)
op, resp, rpc_id = await self.write_rpc_request(ClusterOp.PING, req)
return (resp.id, resp.shard_mask_list)
def get_root_block_by_hash(self, h) -> Optional[RootBlock]:
key = b"rblock_" + h
if key in self.rblock_cache:
return self.rblock_cache[key]
raw_block = self.db.get(key, None)
block = raw_block and RootBlock.deserialize(raw_block)
if block is not None:
self.rblock_cache[key] = block
return block
def __recover_from_db(self):
""" Recover the best chain from local database.
"""
Logger.info("Recovering root chain from local database...")
if b"tipHash" not in self.db:
return None
r_hash = self.db.get(b"tipHash")
r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash))
if r_block.header.height <= 0:
return None
# use the parent of the tipHash block as the new tip
# since it's guaranteed to have been accepted by all the shards
# while shards might not have seen the block of tipHash
r_hash = r_block.header.hash_prev_block
r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash))
self.tip_header = r_block.header # type: RootBlockHeader
async def create(**kwargs):
nonlocal now, mock_tip
return RootBlock(
RootBlockHeader(
create_time=now,
extra_data=b"{}",
hash_prev_block=mock_tip.get_hash(),
height=mock_tip.height + 1,
))
def test_submit_work(self):
now, height = 42, 42
doublesha = ConsensusType.POW_DOUBLESHA256
mock_tip = RootBlockHeader(height=height)
block = RootBlock(
RootBlockHeader(create_time=42, extra_data=b"{}", difficulty=5))
async def create(coinbase_addr=None, retry=True):
nonlocal block, mock_tip
ret = copy.deepcopy(block)
ret.header.height = mock_tip.height + 1
ret.header.hash_prev_block = mock_tip.get_hash()
return ret
async def add(block_to_add):
validate_seal(block_to_add.header, doublesha)
self.added_blocks.append(block_to_add)
def tip_getter():
nonlocal mock_tip
return mock_tip
miner = self.miner_gen(doublesha, create, add, tip_getter, remote=True)
async def go():
nonlocal mock_tip
work, _ = await miner.get_work(EMPTY_ADDR, now=now)
self.assertEqual(work.height, 43)
self.assertEqual(work.difficulty, 5)
# submitted block doesn't exist
res = await miner.submit_work(b"lolwut", 0, sha3_256(b""))
self.assertFalse(res)
solver = DoubleSHA256(work)
sol = solver.mine(200, 300).nonce
self.assertGreater(sol, 200) # ensure non-solution is tried
non_sol = sol - 1
# invalid pow proof
res = await miner.submit_work(work.hash, non_sol, sha3_256(b""))
self.assertFalse(res)
# valid submission, but tip updated so should fail
mock_tip.height += 1
res = await miner.submit_work(work.hash, sol, sha3_256(b""))
self.assertFalse(res)
self.assertEqual(miner.work_map, {})
# bring tip back and regenerate the work
mock_tip.height -= 1
work, _ = await miner.get_work(EMPTY_ADDR, now=now)
# valid submission, also check internal state afterwards
res = await miner.submit_work(work.hash, sol, sha3_256(b""))
self.assertTrue(res)
self.assertEqual(miner.work_map, {})
self.assertEqual(len(self.added_blocks), 1)
loop = asyncio.get_event_loop()
loop.run_until_complete(go())
def create_root_block(self) -> RootBlock:
""" Create the genesis root block """
genesis = self._qkc_config.ROOT.GENESIS
header = RootBlockHeader(
version=genesis.VERSION,
height=genesis.HEIGHT,
hash_prev_block=bytes.fromhex(genesis.HASH_PREV_BLOCK),
hash_merkle_root=bytes.fromhex(genesis.HASH_MERKLE_ROOT),
create_time=genesis.TIMESTAMP,
difficulty=genesis.DIFFICULTY,
)
return RootBlock(header=header, minor_block_header_list=[])
def test_sha3sha3(self):
miner = self.miner_gen(ConsensusType.POW_SHA3SHA3, None, None)
block = RootBlock(
RootBlockHeader(create_time=42,
extra_data="{}".encode("utf-8"),
difficulty=5))
# only process one block, which is passed in
miner.input_q.put((None, {}))
miner.mine_sha3sha3(block, miner.input_q, miner.output_q, {})
mined_block = miner.output_q.get()
self.assertEqual(mined_block.header.nonce, 3)
validate_seal(mined_block.header, ConsensusType.POW_SHA3SHA3)
def get_root_block_by_hash(self, h, consistency_check=True):
"""
Consistency check being true means whatever in memory should already have enough
information (no missing root block, minor block etc). Skipping the check by reading
directly from database may have unwanted consequences.
"""
if consistency_check and h not in self.r_header_pool:
return None
raw_block = self.db.get(b"rblock_" + h, None)
if not raw_block:
return None
return RootBlock.deserialize(raw_block)
def __recover_from_db(self):
""" Recover the best chain from local database.
"""
Logger.info("Recovering root chain from local database...")
if b"tipHash" not in self.db:
return None
r_hash = self.db.get(b"tipHash")
r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash))
self.tip_header = r_block.header
while len(self.r_header_pool) < self.max_num_blocks_to_recover:
self.r_header_pool[r_hash] = r_block.header
for m_header in r_block.minor_block_header_list:
self.m_hash_set.add(m_header.get_hash())
if r_block.header.height <= 0:
break
r_hash = r_block.header.hash_prev_block
r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash))
def test_submit_work_with_remote_guardian(self):
now = 42
doublesha = ConsensusType.POW_DOUBLESHA256
block = RootBlock(
RootBlockHeader(
create_time=42,
extra_data=b"{}",
difficulty=1000,
hash_prev_block=RootBlockHeader().get_hash(),
))
priv = ecies.generate_privkey()
async def create(coinbase_addr=None, retry=True):
return block
async def add(block_to_add):
h = block_to_add.header
diff = h.difficulty
if h.verify_signature(priv.public_key):
diff = Guardian.adjust_difficulty(diff, h.height)
validate_seal(block_to_add.header, doublesha, adjusted_diff=diff)
# just with the guardian public key
miner = self.miner_gen(doublesha, create, add, remote=True)
async def go():
for i in range(42, 100):
work, _ = await miner.get_work(EMPTY_ADDR, now=now)
self.assertEqual(work.height, 0)
# remote guardian: diff 1000 -> 1, any number should work
# mimic the sign process of the remote guardian server
block.header.nonce = i
block.header.mixhash = sha3_256(b"")
block.header.sign_with_private_key(priv)
signature = block.header.signature
# reset the signature to the default value
block.header.signature = bytes(65)
# submit the signature through the submit work
res = await miner.submit_work(work.hash, i, sha3_256(b""),
signature)
self.assertTrue(res)
res = await miner.submit_work(work.hash, i, sha3_256(b""),
bytes(65))
self.assertFalse(res)
loop = asyncio.get_event_loop()
loop.run_until_complete(go())
def test_validate_seal_with_adjusted_diff(self):
diff = 1000
block = RootBlock(
RootBlockHeader(create_time=42, difficulty=diff),
tracking_data="{}".encode("utf-8"),
)
block.header.nonce = 0
with self.assertRaises(ValueError):
validate_seal(block.header, ConsensusType.POW_DOUBLESHA256)
# significantly lowering the diff should pass
validate_seal(block.header,
ConsensusType.POW_DOUBLESHA256,
adjusted_diff=1)
def test_sha3sha3(self):
miner = self.miner_gen(ConsensusType.POW_SHA3SHA3, None, None)
block = RootBlock(
RootBlockHeader(create_time=42, difficulty=5),
tracking_data="{}".encode("utf-8"),
)
work = MiningWork(block.header.get_hash_for_mining(), 42, 5)
# only process one block, which is passed in. `None` means termination right after
miner.input_q.put((None, {}))
miner._mine_loop(
ConsensusType.POW_SHA3SHA3, work, {}, miner.input_q, miner.output_q
)
mined_res = miner.output_q.get()
self.assertEqual(mined_res.nonce, 8)
block.header.nonce = mined_res.nonce
validate_seal(block.header, ConsensusType.POW_SHA3SHA3)
def test_qkchash(self):
miner = self.miner_gen(ConsensusType.POW_QKCHASH, None, None)
block = RootBlock(
RootBlockHeader(create_time=42, difficulty=5),
tracking_data="{}".encode("utf-8"),
)
work = MiningWork(block.header.get_hash_for_mining(), 42, 5)
# only process one block, which is passed in. `None` means termination right after
miner.input_q.put((None, {}))
miner.mine_loop(
work,
{"consensus_type": ConsensusType.POW_QKCHASH},
miner.input_q,
miner.output_q,
)
mined_res = miner.output_q.get()
block.header.nonce = mined_res.nonce
block.header.mixhash = mined_res.mixhash
validate_seal(block.header, ConsensusType.POW_QKCHASH)
def test_submit_work(self):
now = 42
block = RootBlock(
RootBlockHeader(create_time=42, extra_data=b"{}", difficulty=2))
async def create():
return block
async def add(block_to_add):
self.added_blocks.append(block_to_add)
miner = self.miner_gen(ConsensusType.POW_SHA3SHA3,
create,
add,
remote=True)
async def go():
work = await miner.get_work(now=now)
self.assertEqual(work.height, 0)
self.assertEqual(work.difficulty, 2)
# submitted block doesn't exist
res = await miner.submit_work(b"lolwut", 0, sha3_256(b""))
self.assertFalse(res)
solver = DoubleSHA256(block)
mined = solver.mine(100, 200)
self.assertTrue(mined)
sol = int.from_bytes(solver.nonce_found, byteorder="big")
self.assertGreater(sol, 100) # ensure non-solution is tried
non_sol = sol - 1
# invalid pow proof
res = await miner.submit_work(work.hash, non_sol, sha3_256(b""))
self.assertFalse(res)
# valid submission, also check internal state afterwards
res = await miner.submit_work(work.hash, sol, sha3_256(b""))
self.assertTrue(res)
self.assertEqual(miner.work_map, {})
self.assertEqual(len(self.added_blocks), 1)
self.assertIsNone(miner.current_work)
loop = asyncio.get_event_loop()
loop.run_until_complete(go())
def test_submit_work(self):
now = 42
doublesha = ConsensusType.POW_DOUBLESHA256
block = RootBlock(
RootBlockHeader(create_time=42, extra_data=b"{}", difficulty=5))
async def create(retry=True):
return block
async def add(block_to_add):
validate_seal(block_to_add.header, doublesha)
self.added_blocks.append(block_to_add)
miner = self.miner_gen(doublesha, create, add, remote=True)
async def go():
work, _ = await miner.get_work(now=now)
self.assertEqual(work.height, 0)
self.assertEqual(work.difficulty, 5)
# submitted block doesn't exist
res = await miner.submit_work(b"lolwut", 0, sha3_256(b""))
self.assertFalse(res)
solver = DoubleSHA256(work)
sol = solver.mine(200, 300).nonce
self.assertGreater(sol, 200) # ensure non-solution is tried
non_sol = sol - 1
# invalid pow proof
res = await miner.submit_work(work.hash, non_sol, sha3_256(b""))
self.assertFalse(res)
# valid submission, also check internal state afterwards
res = await miner.submit_work(work.hash, sol, sha3_256(b""))
self.assertTrue(res)
self.assertEqual(miner.work_map, {})
self.assertEqual(len(self.added_blocks), 1)
self.assertIsNone(miner.current_work)
loop = asyncio.get_event_loop()
loop.run_until_complete(go())
def test_submit_work_with_guardian(self):
now = 42
doublesha = ConsensusType.POW_DOUBLESHA256
block = RootBlock(
RootBlockHeader(
create_time=42,
extra_data=b"{}",
difficulty=1000,
hash_prev_block=RootBlockHeader().get_hash(),
))
priv = ecies.generate_privkey()
async def create(coinbase_addr=None, retry=True):
return block
async def add(block_to_add):
h = block_to_add.header
diff = h.difficulty
if h.verify_signature(priv.public_key):
diff = Guardian.adjust_difficulty(diff, h.height)
validate_seal(block_to_add.header, doublesha, adjusted_diff=diff)
miner = self.miner_gen(doublesha,
create,
add,
remote=True,
guardian_private_key=priv)
async def go():
for i in range(42, 100):
work, _ = await miner.get_work(EMPTY_ADDR, now=now)
self.assertEqual(work.height, 0)
# guardian: diff 1000 -> 1, any number should work
res = await miner.submit_work(work.hash, i, sha3_256(b""))
self.assertTrue(res)
loop = asyncio.get_event_loop()
loop.run_until_complete(go())
async def addBlock(self, branch, block_data):
if branch == 0:
block = RootBlock.deserialize(block_data)
return await self.master.add_root_block_from_miner(block)
return await self.master.add_raw_minor_block(Branch(branch),
block_data)
async def dummy_create_block_async() -> Optional[RootBlock]:
if len(TestMiner.added_blocks) >= 5:
return None # stop the game
return RootBlock(RootBlockHeader(create_time=int(time.time()), extra_data="{}".encode("utf-8")))
def test_getNextBlockToMine_and_addBlock(self):
id1 = Identity.create_random_identity()
acc1 = Address.create_from_identity(id1, full_shard_id=0)
acc3 = Address.create_random_account(full_shard_id=1)
with ClusterContext(1, acc1) as clusters, jrpc_server_context(
clusters[0].master):
slaves = clusters[0].slave_list
# Expect to mine root that confirms the genesis minor blocks
response = send_request("getNextBlockToMine",
"0x" + acc1.serialize().hex(), "0x0")
self.assertTrue(response["isRootBlock"])
block = RootBlock.deserialize(
bytes.fromhex(response["blockData"][2:]))
self.assertEqual(block.header.height, 1)
self.assertEqual(len(block.minor_block_header_list), 2)
self.assertEqual(block.minor_block_header_list[0].height, 0)
self.assertEqual(block.minor_block_header_list[1].height, 0)
send_request("addBlock", "0x0", response["blockData"])
tx = create_transfer_transaction(
shard_state=clusters[0].get_shard_state(0),
key=id1.get_key(),
from_address=acc1,
to_address=acc3,
value=14,
gas=opcodes.GTXXSHARDCOST + opcodes.GTXCOST,
)
self.assertTrue(slaves[0].add_tx(tx))
# Expect to mine shard 0 since it has one tx
response = send_request("getNextBlockToMine",
"0x" + acc1.serialize().hex(), "0x0")
self.assertFalse(response["isRootBlock"])
block1 = MinorBlock.deserialize(
bytes.fromhex(response["blockData"][2:]))
self.assertEqual(block1.header.branch.value, 0b10)
self.assertTrue(
send_request("addBlock", "0x2", response["blockData"]))
self.assertEqual(
clusters[0].get_shard_state(1).get_balance(acc3.recipient, 0),
0)
# Expect to mine shard 1 due to proof-of-progress
response = send_request("getNextBlockToMine",
"0x" + acc1.serialize().hex(), "0x0")
self.assertFalse(response["isRootBlock"])
block2 = MinorBlock.deserialize(
bytes.fromhex(response["blockData"][2:]))
self.assertEqual(block2.header.branch.value, 0b11)
self.assertTrue(
send_request("addBlock", "0x3", response["blockData"]))
# Expect to mine root
response = send_request("getNextBlockToMine",
"0x" + acc1.serialize().hex(), "0x0")
self.assertTrue(response["isRootBlock"])
block = RootBlock.deserialize(
bytes.fromhex(response["blockData"][2:]))
self.assertEqual(block.header.height, 2)
self.assertEqual(len(block.minor_block_header_list), 2)
self.assertEqual(block.minor_block_header_list[0], block1.header)
self.assertEqual(block.minor_block_header_list[1], block2.header)
send_request("addBlock", "0x0", response["blockData"])
self.assertEqual(
clusters[0].get_shard_state(1).get_balance(acc3.recipient, 0),
0)
# Expect to mine shard 1 for the gas on xshard tx to acc3
response = send_request("getNextBlockToMine",
"0x" + acc1.serialize().hex(), "0x0")
self.assertFalse(response["isRootBlock"])
block3 = MinorBlock.deserialize(
bytes.fromhex(response["blockData"][2:]))
self.assertEqual(block3.header.branch.value, 0b11)
self.assertTrue(
send_request("addBlock", "0x3", response["blockData"]))
# Expect withdrawTo is included in acc3's balance
resp = send_request("getBalance", "0x" + acc3.serialize().hex())
self.assertEqual(resp["branch"], "0x3")
self.assertEqual(resp["balance"], "0xe")
async def create(retry=True):
nonlocal now
return RootBlock(RootBlockHeader(create_time=now,
extra_data=b"{}"))
def get_root_block_by_hash(self, h):
if h not in self.r_header_pool:
return None
return RootBlock.deserialize(self.db.get(b"rblock_" + h))