async def test_deadlock(self, tmp_dir, db_version):
"""
This test was added because the store was deadlocking in certain situations, when fetching and
adding blocks repeatedly. The issue was patched.
"""
blocks = bt.get_consecutive_blocks(10)
async with DBConnection(db_version) as wrapper, DBConnection(
db_version) as wrapper_2:
store = await BlockStore.create(wrapper)
coin_store_2 = await CoinStore.create(wrapper_2)
store_2 = await BlockStore.create(wrapper_2)
hint_store = await HintStore.create(wrapper_2)
bc = await Blockchain.create(coin_store_2, store_2, test_constants,
hint_store, tmp_dir)
block_records = []
for block in blocks:
await bc.receive_block(block)
block_records.append(bc.block_record(block.header_hash))
tasks = []
for i in range(10000):
rand_i = random.randint(0, 9)
if random.random() < 0.5:
tasks.append(
asyncio.create_task(
store.add_full_block(blocks[rand_i].header_hash,
blocks[rand_i],
block_records[rand_i])))
if random.random() < 0.5:
tasks.append(
asyncio.create_task(
store.get_full_block(blocks[rand_i].header_hash)))
await asyncio.gather(*tasks)
async def test_rollback2(self, tmp_dir, db_version):
async with DBConnection(db_version) as db_wrapper:
await setup_db(db_wrapper)
await setup_chain(db_wrapper, 10, ses_every=2)
height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
assert height_map.get_ses(0) == gen_ses(0)
assert height_map.get_ses(2) == gen_ses(2)
assert height_map.get_ses(4) == gen_ses(4)
assert height_map.get_ses(6) == gen_ses(6)
assert height_map.get_ses(8) == gen_ses(8)
assert height_map.get_hash(6) == gen_block_hash(6)
height_map.rollback(6)
assert height_map.get_hash(6) == gen_block_hash(6)
assert height_map.get_ses(0) == gen_ses(0)
assert height_map.get_ses(2) == gen_ses(2)
assert height_map.get_ses(4) == gen_ses(4)
assert height_map.get_ses(6) == gen_ses(6)
with pytest.raises(KeyError) as _:
height_map.get_ses(8)
async def test_rollback(self, cache_size: uint32, db_version):
blocks = bt.get_consecutive_blocks(20)
async with DBConnection(db_version) as db_wrapper:
coin_store = await CoinStore.create(db_wrapper,
cache_size=uint32(cache_size))
records: List[CoinRecord] = []
for block in blocks:
if block.is_transaction_block():
removals: List[bytes32] = []
additions: List[Coin] = []
if block.is_transaction_block():
assert block.foliage_transaction_block is not None
await coin_store.new_block(
block.height,
block.foliage_transaction_block.timestamp,
block.get_included_reward_coins(),
additions,
removals,
)
coins = block.get_included_reward_coins()
records = [
await coin_store.get_coin_record(coin.name())
for coin in coins
]
await coin_store._set_spent([r.name for r in records],
block.height)
records = [
await coin_store.get_coin_record(coin.name())
for coin in coins
]
for record in records:
assert record is not None
assert record.spent
assert record.spent_block_index == block.height
reorg_index = 8
await coin_store.rollback_to_block(reorg_index)
for block in blocks:
if block.is_transaction_block():
coins = block.get_included_reward_coins()
records = [
await coin_store.get_coin_record(coin.name())
for coin in coins
]
if block.height <= reorg_index:
for record in records:
assert record is not None
assert record.spent
else:
for record in records:
assert record is None
async def test_get_puzzle_hash(self, cache_size: uint32, tmp_dir, db_version):
async with DBConnection(db_version) as db_wrapper:
num_blocks = 20
farmer_ph = 32 * b"0"
pool_ph = 32 * b"1"
# TODO: address hint error and remove ignore
# error: Argument "farmer_reward_puzzle_hash" to "get_consecutive_blocks" of "BlockTools" has
# incompatible type "bytes"; expected "Optional[bytes32]" [arg-type]
# error: Argument "pool_reward_puzzle_hash" to "get_consecutive_blocks" of "BlockTools" has
# incompatible type "bytes"; expected "Optional[bytes32]" [arg-type]
blocks = bt.get_consecutive_blocks(
num_blocks,
farmer_reward_puzzle_hash=farmer_ph, # type: ignore[arg-type]
pool_reward_puzzle_hash=pool_ph, # type: ignore[arg-type]
guarantee_transaction_block=True,
)
coin_store = await CoinStore.create(db_wrapper, cache_size=uint32(cache_size))
store = await BlockStore.create(db_wrapper)
hint_store = await HintStore.create(db_wrapper)
b: Blockchain = await Blockchain.create(coin_store, store, test_constants, hint_store, tmp_dir, 2)
for block in blocks:
await _validate_and_add_block(b, block)
peak = b.get_peak()
assert peak is not None
assert peak.height == num_blocks - 1
coins_farmer = await coin_store.get_coin_records_by_puzzle_hash(True, pool_ph)
coins_pool = await coin_store.get_coin_records_by_puzzle_hash(True, farmer_ph)
assert len(coins_farmer) == num_blocks - 2
assert len(coins_pool) == num_blocks - 2
b.shut_down()
async def test_num_unspent(self, db_version):
blocks = bt.get_consecutive_blocks(37, [])
expect_unspent = 0
test_excercised = False
async with DBConnection(db_version) as db_wrapper:
coin_store = await CoinStore.create(db_wrapper)
for block in blocks:
if not block.is_transaction_block():
continue
if block.is_transaction_block():
assert block.foliage_transaction_block is not None
removals: List[bytes32] = []
additions: List[Coin] = []
await coin_store.new_block(
block.height,
block.foliage_transaction_block.timestamp,
block.get_included_reward_coins(),
additions,
removals,
)
expect_unspent += len(block.get_included_reward_coins())
assert await coin_store.num_unspent() == expect_unspent
test_excercised = expect_unspent > 0
assert test_excercised
async def test_basic_store(self, db_version):
async with DBConnection(db_version) as db_wrapper:
hint_store = await HintStore.create(db_wrapper)
hint_0 = 32 * b"\0"
hint_1 = 32 * b"\1"
not_existing_hint = 32 * b"\3"
coin_id_0 = 32 * b"\4"
coin_id_1 = 32 * b"\5"
coin_id_2 = 32 * b"\6"
hints = [(coin_id_0, hint_0), (coin_id_1, hint_0),
(coin_id_2, hint_1)]
await hint_store.add_hints(hints)
await db_wrapper.commit_transaction()
coins_for_hint_0 = await hint_store.get_coin_ids(hint_0)
assert coin_id_0 in coins_for_hint_0
assert coin_id_1 in coins_for_hint_0
coins_for_hint_1 = await hint_store.get_coin_ids(hint_1)
assert coin_id_2 in coins_for_hint_1
coins_for_non_hint = await hint_store.get_coin_ids(
not_existing_hint)
assert coins_for_non_hint == []
async def test_basic_reorg(self, cache_size: uint32, tmp_dir, db_version):
async with DBConnection(db_version) as db_wrapper:
initial_block_count = 30
reorg_length = 15
blocks = bt.get_consecutive_blocks(initial_block_count)
coin_store = await CoinStore.create(db_wrapper, cache_size=uint32(cache_size))
store = await BlockStore.create(db_wrapper)
hint_store = await HintStore.create(db_wrapper)
b: Blockchain = await Blockchain.create(coin_store, store, test_constants, hint_store, tmp_dir, 2)
try:
records: List[Optional[CoinRecord]] = []
for block in blocks:
await _validate_and_add_block(b, block)
peak = b.get_peak()
assert peak is not None
assert peak.height == initial_block_count - 1
for c, block in enumerate(blocks):
if block.is_transaction_block():
coins = block.get_included_reward_coins()
records = [await coin_store.get_coin_record(coin.name()) for coin in coins]
for record in records:
assert record is not None
assert not record.spent
assert record.confirmed_block_index == block.height
assert record.spent_block_index == 0
blocks_reorg_chain = bt.get_consecutive_blocks(
reorg_length, blocks[: initial_block_count - 10], seed=b"2"
)
for reorg_block in blocks_reorg_chain:
if reorg_block.height < initial_block_count - 10:
await _validate_and_add_block(
b, reorg_block, expected_result=ReceiveBlockResult.ALREADY_HAVE_BLOCK
)
elif reorg_block.height < initial_block_count:
await _validate_and_add_block(
b, reorg_block, expected_result=ReceiveBlockResult.ADDED_AS_ORPHAN
)
elif reorg_block.height >= initial_block_count:
await _validate_and_add_block(b, reorg_block, expected_result=ReceiveBlockResult.NEW_PEAK)
if reorg_block.is_transaction_block():
coins = reorg_block.get_included_reward_coins()
records = [await coin_store.get_coin_record(coin.name()) for coin in coins]
for record in records:
assert record is not None
assert not record.spent
assert record.confirmed_block_index == reorg_block.height
assert record.spent_block_index == 0
peak = b.get_peak()
assert peak is not None
assert peak.height == initial_block_count - 10 + reorg_length - 1
finally:
b.shut_down()
async def test_block_store(self, tmp_dir, db_version):
assert sqlite3.threadsafety == 1
blocks = bt.get_consecutive_blocks(10)
async with DBConnection(db_version) as db_wrapper, DBConnection(
db_version) as db_wrapper_2:
# Use a different file for the blockchain
coin_store_2 = await CoinStore.create(db_wrapper_2)
store_2 = await BlockStore.create(db_wrapper_2)
hint_store = await HintStore.create(db_wrapper_2)
bc = await Blockchain.create(coin_store_2, store_2, test_constants,
hint_store, tmp_dir)
store = await BlockStore.create(db_wrapper)
await BlockStore.create(db_wrapper_2)
# Save/get block
for block in blocks:
await bc.receive_block(block)
block_record = bc.block_record(block.header_hash)
block_record_hh = block_record.header_hash
await store.add_full_block(block.header_hash, block,
block_record)
await store.add_full_block(block.header_hash, block,
block_record)
assert block == await store.get_full_block(block.header_hash)
assert block == await store.get_full_block(block.header_hash)
assert block_record == (
await store.get_block_record(block_record_hh))
await store.set_in_chain([(block_record.header_hash, )])
await store.set_peak(block_record.header_hash)
await store.set_peak(block_record.header_hash)
assert len(await store.get_full_blocks_at([1])) == 1
assert len(await store.get_full_blocks_at([0])) == 1
assert len(await store.get_full_blocks_at([100])) == 0
# Get blocks
block_record_records = await store.get_block_records_in_range(
0, 0xFFFFFFFF)
assert len(block_record_records) == len(blocks)
async def test_restore_extend(self, tmp_dir, db_version):
# test the case where the cache has fewer blocks than the DB, and that
# we correctly load all the missing blocks from the DB to update the
# cache
async with DBConnection(db_version) as db_wrapper:
await setup_db(db_wrapper)
await setup_chain(db_wrapper, 2000, ses_every=20)
height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
for height in reversed(range(2000)):
assert height_map.contains_height(height)
assert height_map.get_hash(height) == gen_block_hash(height)
if (height % 20) == 0:
assert height_map.get_ses(height) == gen_ses(height)
else:
with pytest.raises(KeyError) as _:
height_map.get_ses(height)
await height_map.maybe_flush()
del height_map
async with DBConnection(db_version) as db_wrapper:
await setup_db(db_wrapper)
# add 2000 blocks to the chain
await setup_chain(db_wrapper, 4000, ses_every=20)
height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
# now make sure we have the complete chain, height 0 -> 4000
for height in reversed(range(4000)):
assert height_map.contains_height(height)
assert height_map.get_hash(height) == gen_block_hash(height)
if (height % 20) == 0:
assert height_map.get_ses(height) == gen_ses(height)
else:
with pytest.raises(KeyError) as _:
height_map.get_ses(height)
async def test_height_to_hash_long_chain(self, tmp_dir, db_version):
async with DBConnection(db_version) as db_wrapper:
await setup_db(db_wrapper)
await setup_chain(db_wrapper, 10000)
height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
for height in reversed(range(1000)):
assert height_map.contains_height(height)
for height in reversed(range(10000)):
assert height_map.get_hash(height) == gen_block_hash(height)
async def test_height_to_hash_with_orphans(self, tmp_dir, db_version):
async with DBConnection(db_version) as db_wrapper:
await setup_db(db_wrapper)
await setup_chain(db_wrapper, 10)
# set up two separate chains, but without the peak
await setup_chain(db_wrapper, 10, chain_id=1)
await setup_chain(db_wrapper, 10, chain_id=2)
height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
for height in range(10):
assert height_map.get_hash(height) == gen_block_hash(height)
async def test_save_restore(self, tmp_dir, db_version):
async with DBConnection(db_version) as db_wrapper:
await setup_db(db_wrapper)
await setup_chain(db_wrapper, 10000, ses_every=20)
height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
for height in reversed(range(10000)):
assert height_map.contains_height(height)
assert height_map.get_hash(height) == gen_block_hash(height)
if (height % 20) == 0:
assert height_map.get_ses(height) == gen_ses(height)
else:
with pytest.raises(KeyError) as _:
height_map.get_ses(height)
await height_map.maybe_flush()
del height_map
# To ensure we're actually loading from cache, and not the DB, clear
# the table (but we still need the peak). We need at least 20 blocks
# in the DB since we keep loading until we find a match of both hash
# and sub epoch summary. In this test we have a sub epoch summary
# every 20 blocks, so we generate the 30 last blocks only
if db_version == 2:
await db_wrapper.db.execute("DROP TABLE full_blocks")
else:
await db_wrapper.db.execute("DROP TABLE block_records")
await setup_db(db_wrapper)
await setup_chain(db_wrapper,
10000,
ses_every=20,
start_height=9970)
height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
for height in reversed(range(10000)):
assert height_map.contains_height(height)
assert height_map.get_hash(height) == gen_block_hash(height)
if (height % 20) == 0:
assert height_map.get_ses(height) == gen_ses(height)
else:
with pytest.raises(KeyError) as _:
height_map.get_ses(height)
async def test_update_ses(self, tmp_dir, db_version):
async with DBConnection(db_version) as db_wrapper:
await setup_db(db_wrapper)
await setup_chain(db_wrapper, 10)
# orphan blocks
await setup_chain(db_wrapper, 10, chain_id=1)
height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
with pytest.raises(KeyError) as _:
height_map.get_ses(10)
height_map.update_height(10, gen_block_hash(10), gen_ses(10))
assert height_map.get_ses(10) == gen_ses(10)
assert height_map.get_hash(10) == gen_block_hash(10)
async def test_rollback(self, tmp_dir):
blocks = bt.get_consecutive_blocks(10)
async with DBConnection(2) as db_wrapper:
# Use a different file for the blockchain
coin_store = await CoinStore.create(db_wrapper)
block_store = await BlockStore.create(db_wrapper)
hint_store = await HintStore.create(db_wrapper)
bc = await Blockchain.create(coin_store, block_store,
test_constants, hint_store, tmp_dir)
# insert all blocks
count = 0
for block in blocks:
await bc.receive_block(block)
count += 1
ret = await block_store.get_random_not_compactified(count)
assert len(ret) == count
# make sure all block heights are unique
assert len(set(ret)) == count
for block in blocks:
async with db_wrapper.db.execute(
"SELECT in_main_chain FROM full_blocks WHERE header_hash=?",
(block.header_hash, )) as cursor:
rows = await cursor.fetchall()
assert len(rows) == 1
assert rows[0][0]
await block_store.rollback(5)
count = 0
for block in blocks:
async with db_wrapper.db.execute(
"SELECT in_main_chain FROM full_blocks WHERE header_hash=? ORDER BY height",
(block.header_hash, )) as cursor:
rows = await cursor.fetchall()
print(count, rows)
assert len(rows) == 1
assert rows[0][0] == (count <= 5)
count += 1
async def test_height_to_hash_update(self, tmp_dir, db_version):
async with DBConnection(db_version) as db_wrapper:
await setup_db(db_wrapper)
await setup_chain(db_wrapper, 10)
# orphan blocks
await setup_chain(db_wrapper, 10, chain_id=1)
height_map = await BlockHeightMap.create(tmp_dir, db_wrapper)
for height in range(10):
assert height_map.get_hash(height) == gen_block_hash(height)
height_map.update_height(10, gen_block_hash(100), None)
for height in range(9):
assert height_map.get_hash(height) == gen_block_hash(height)
assert height_map.get_hash(10) == gen_block_hash(100)
async def test_basic_coin_store(self, cache_size: uint32, db_version):
wallet_a = WALLET_A
reward_ph = wallet_a.get_new_puzzlehash()
# Generate some coins
blocks = bt.get_consecutive_blocks(
10,
[],
farmer_reward_puzzle_hash=reward_ph,
pool_reward_puzzle_hash=reward_ph,
)
coins_to_spend: List[Coin] = []
for block in blocks:
if block.is_transaction_block():
for coin in block.get_included_reward_coins():
if coin.puzzle_hash == reward_ph:
coins_to_spend.append(coin)
spend_bundle = wallet_a.generate_signed_transaction(
uint64(1000), wallet_a.get_new_puzzlehash(), coins_to_spend[0])
async with DBConnection(db_version) as db_wrapper:
coin_store = await CoinStore.create(db_wrapper,
cache_size=cache_size)
blocks = bt.get_consecutive_blocks(
10,
blocks,
farmer_reward_puzzle_hash=reward_ph,
pool_reward_puzzle_hash=reward_ph,
transaction_data=spend_bundle,
)
# Adding blocks to the coin store
should_be_included_prev: Set[Coin] = set()
should_be_included: Set[Coin] = set()
for block in blocks:
farmer_coin, pool_coin = get_future_reward_coins(block)
should_be_included.add(farmer_coin)
should_be_included.add(pool_coin)
if block.is_transaction_block():
if block.transactions_generator is not None:
block_gen: BlockGenerator = BlockGenerator(
block.transactions_generator, [])
npc_result = get_name_puzzle_conditions(
block_gen,
bt.constants.MAX_BLOCK_COST_CLVM,
cost_per_byte=bt.constants.COST_PER_BYTE,
mempool_mode=False,
)
tx_removals, tx_additions = tx_removals_and_additions(
npc_result.npc_list)
else:
tx_removals, tx_additions = [], []
assert block.get_included_reward_coins(
) == should_be_included_prev
if block.is_transaction_block():
assert block.foliage_transaction_block is not None
await coin_store.new_block(
block.height,
block.foliage_transaction_block.timestamp,
block.get_included_reward_coins(),
tx_additions,
tx_removals,
)
if block.height != 0:
with pytest.raises(Exception):
await coin_store.new_block(
block.height,
block.foliage_transaction_block.timestamp,
block.get_included_reward_coins(),
tx_additions,
tx_removals,
)
for expected_coin in should_be_included_prev:
# Check that the coinbase rewards are added
record = await coin_store.get_coin_record(
expected_coin.name())
assert record is not None
assert not record.spent
assert record.coin == expected_coin
for coin_name in tx_removals:
# Check that the removed coins are set to spent
record = await coin_store.get_coin_record(coin_name)
assert record.spent
for coin in tx_additions:
# Check that the added coins are added
record = await coin_store.get_coin_record(coin.name())
assert not record.spent
assert coin == record.coin
should_be_included_prev = should_be_included.copy()
should_be_included = set()