async def test_block_store(self):
assert sqlite3.threadsafety == 1
blocks = bt.get_consecutive_blocks(10)
db_filename = Path("blockchain_test.db")
db_filename_2 = Path("blockchain_test2.db")
if db_filename.exists():
db_filename.unlink()
if db_filename_2.exists():
db_filename_2.unlink()
connection = await aiosqlite.connect(db_filename)
connection_2 = await aiosqlite.connect(db_filename_2)
db_wrapper = DBWrapper(connection)
db_wrapper_2 = DBWrapper(connection_2)
# Use a different file for the blockchain
coin_store_2 = await CoinStore.create(db_wrapper_2)
store_2 = await BlockStore.create(db_wrapper_2)
bc = await Blockchain.create(coin_store_2, store_2, test_constants)
store = await BlockStore.create(db_wrapper)
await BlockStore.create(db_wrapper_2)
try:
# Save/get block
for block in blocks:
await bc.receive_block(block)
block_record = bc.block_record(block.header_hash)
block_record_hh = block_record.header_hash
await store.add_full_block(block.header_hash, block, block_record)
await store.add_full_block(block.header_hash, block, block_record)
assert block == await store.get_full_block(block.header_hash)
assert block == await store.get_full_block(block.header_hash)
assert block_record == (await store.get_block_record(block_record_hh))
await store.set_peak(block_record.header_hash)
await store.set_peak(block_record.header_hash)
assert len(await store.get_full_blocks_at([1])) == 1
assert len(await store.get_full_blocks_at([0])) == 1
assert len(await store.get_full_blocks_at([100])) == 0
# Get blocks
block_record_records = await store.get_block_records()
assert len(block_record_records[0]) == len(blocks)
# Peak is correct
assert block_record_records[1] == blocks[-1].header_hash
except Exception:
await connection.close()
await connection_2.close()
db_filename.unlink()
db_filename_2.unlink()
raise
await connection.close()
await connection_2.close()
db_filename.unlink()
db_filename_2.unlink()
async def test_set_spent(self):
blocks = bt.get_consecutive_blocks(9, [])
db_path = Path("fndb_test.db")
if db_path.exists():
db_path.unlink()
connection = await aiosqlite.connect(db_path)
db_wrapper = DBWrapper(connection)
coin_store = await CoinStore.create(db_wrapper)
# Save/get block
for block in blocks:
if block.is_transaction_block():
await coin_store.new_block(block)
coins = block.get_included_reward_coins()
records = [
await coin_store.get_coin_record(coin.name())
for coin in coins
]
for record in records:
await coin_store._set_spent(record.coin.name(),
block.height)
records = [
await coin_store.get_coin_record(coin.name())
for coin in coins
]
for record in records:
assert record.spent
assert record.spent_block_index == block.height
await connection.close()
Path("fndb_test.db").unlink()
async def test_get_puzzle_hash(self):
for cache_size in [0, 10, 100000]:
num_blocks = 20
farmer_ph = 32 * b"0"
pool_ph = 32 * b"1"
blocks = bt.get_consecutive_blocks(
num_blocks,
farmer_reward_puzzle_hash=farmer_ph,
pool_reward_puzzle_hash=pool_ph,
guarantee_transaction_block=True,
)
db_path = Path("blockchain_test.db")
if db_path.exists():
db_path.unlink()
connection = await aiosqlite.connect(db_path)
db_wrapper = DBWrapper(connection)
coin_store = await CoinStore.create(db_wrapper, cache_size=uint32(cache_size))
store = await BlockStore.create(db_wrapper)
b: Blockchain = await Blockchain.create(coin_store, store, test_constants)
for block in blocks:
res, err, _ = await b.receive_block(block)
assert err is None
assert res == ReceiveBlockResult.NEW_PEAK
assert b.get_peak().height == num_blocks - 1
coins_farmer = await coin_store.get_coin_records_by_puzzle_hash(True, pool_ph)
coins_pool = await coin_store.get_coin_records_by_puzzle_hash(True, farmer_ph)
assert len(coins_farmer) == num_blocks - 2
assert len(coins_pool) == num_blocks - 2
await connection.close()
Path("blockchain_test.db").unlink()
b.shut_down()
async def test_store():
db_filename = Path("wallet_user_store_test.db")
if db_filename.exists():
db_filename.unlink()
db_connection = await aiosqlite.connect(db_filename)
db_wrapper = DBWrapper(db_connection)
store = await WalletUserStore.create(db_wrapper)
try:
await store.init_wallet()
wallet = None
for i in range(1, 5):
assert (await store.get_last_wallet()).id == i
wallet = await store.create_wallet("CAT_WALLET", WalletType.CAT,
"abc")
assert wallet.id == i + 1
assert wallet.id == 5
for i in range(2, 6):
await store.delete_wallet(i, in_transaction=False)
assert (await store.get_last_wallet()).id == 1
wallet = await store.create_wallet("CAT_WALLET", WalletType.CAT, "abc")
# Due to autoincrement, we don't reuse IDs
assert (await store.get_last_wallet()).id == 6
assert wallet.id == 6
assert (await store.get_wallet_by_id(7)) is None
assert (await store.get_wallet_by_id(6)) == wallet
assert await store.get_last_wallet() == wallet
finally:
await db_connection.close()
db_filename.unlink()
async def test_set_spent(self):
blocks = bt.get_consecutive_blocks(9, [])
for cache_size in [0, 10, 100000]:
db_path = Path("fndb_test.db")
if db_path.exists():
db_path.unlink()
connection = await aiosqlite.connect(db_path)
db_wrapper = DBWrapper(connection)
coin_store = await CoinStore.create(db_wrapper, cache_size=uint32(cache_size))
# Save/get block
for block in blocks:
if block.is_transaction_block():
removals, additions = [], []
await coin_store.new_block(block, additions, removals)
coins = block.get_included_reward_coins()
records = [await coin_store.get_coin_record(coin.name()) for coin in coins]
for record in records:
await coin_store._set_spent(record.coin.name(), block.height)
with pytest.raises(AssertionError):
await coin_store._set_spent(record.coin.name(), block.height)
records = [await coin_store.get_coin_record(coin.name()) for coin in coins]
for record in records:
assert record.spent
assert record.spent_block_index == block.height
await connection.close()
Path("fndb_test.db").unlink()
async def test_deadlock(self):
"""
This test was added because the store was deadlocking in certain situations, when fetching and
adding blocks repeatedly. The issue was patched.
"""
blocks = bt.get_consecutive_blocks(10)
db_filename = Path("blockchain_test.db")
db_filename_2 = Path("blockchain_test2.db")
if db_filename.exists():
db_filename.unlink()
if db_filename_2.exists():
db_filename_2.unlink()
connection = await aiosqlite.connect(db_filename)
connection_2 = await aiosqlite.connect(db_filename_2)
wrapper = DBWrapper(connection)
wrapper_2 = DBWrapper(connection_2)
store = await BlockStore.create(wrapper)
coin_store_2 = await CoinStore.create(wrapper_2)
store_2 = await BlockStore.create(wrapper_2)
bc = await Blockchain.create(coin_store_2, store_2, test_constants)
block_records = []
for block in blocks:
await bc.receive_block(block)
block_records.append(bc.block_record(block.header_hash))
tasks = []
for i in range(10000):
rand_i = random.randint(0, 9)
if random.random() < 0.5:
tasks.append(
asyncio.create_task(
store.add_full_block(blocks[rand_i].header_hash,
blocks[rand_i],
block_records[rand_i])))
if random.random() < 0.5:
tasks.append(
asyncio.create_task(
store.get_full_block(blocks[rand_i].header_hash)))
await asyncio.gather(*tasks)
await connection.close()
await connection_2.close()
db_filename.unlink()
db_filename_2.unlink()
async def create_ram_blockchain(
consensus_constants: ConsensusConstants
) -> Tuple[aiosqlite.Connection, Blockchain]:
connection = await aiosqlite.connect(":memory:")
db_wrapper = DBWrapper(connection)
block_store = await BlockStore.create(db_wrapper)
coin_store = await CoinStore.create(db_wrapper)
blockchain = await Blockchain.create(coin_store, block_store,
consensus_constants)
return connection, blockchain
async def create(cls):
self = cls()
self.connection = await aiosqlite.connect(":memory:")
coin_store = await CoinStore.create(DBWrapper(self.connection))
self.mempool_manager = MempoolManager(coin_store, DEFAULT_CONSTANTS)
self.block_records = []
self.blocks = []
self.timestamp = DEFAULT_CONSTANTS.INITIAL_FREEZE_END_TIMESTAMP + 1
self.block_height = 0
return self
async def test_rollback(self):
blocks = bt.get_consecutive_blocks(20)
for cache_size in [0, 10, 100000]:
db_path = Path("fndb_test.db")
if db_path.exists():
db_path.unlink()
connection = await aiosqlite.connect(db_path)
db_wrapper = DBWrapper(connection)
coin_store = await CoinStore.create(db_wrapper,
cache_size=uint32(cache_size))
for block in blocks:
if block.is_transaction_block():
removals, additions = [], []
await coin_store.new_block(block, additions, removals)
coins = block.get_included_reward_coins()
records: List[Optional[CoinRecord]] = [
await coin_store.get_coin_record(coin.name())
for coin in coins
]
for record in records:
await coin_store._set_spent(record.coin.name(),
block.height)
records: List[Optional[CoinRecord]] = [
await coin_store.get_coin_record(coin.name())
for coin in coins
]
for record in records:
assert record.spent
assert record.spent_block_index == block.height
reorg_index = 8
await coin_store.rollback_to_block(reorg_index)
for block in blocks:
if block.is_transaction_block():
coins = block.get_included_reward_coins()
records: List[Optional[CoinRecord]] = [
await coin_store.get_coin_record(coin.name())
for coin in coins
]
if block.height <= reorg_index:
for record in records:
assert record is not None
assert record.spent
else:
for record in records:
assert record is None
await connection.close()
Path("fndb_test.db").unlink()
async def create_blockchain(constants: ConsensusConstants):
db_path = Path("blockchain_test.db")
if db_path.exists():
db_path.unlink()
connection = await aiosqlite.connect(db_path)
wrapper = DBWrapper(connection)
coin_store = await CoinStore.create(wrapper)
store = await BlockStore.create(wrapper)
bc1 = await Blockchain.create(coin_store, store, constants)
assert bc1.get_peak() is None
return bc1, connection, db_path
async def create(cls, defaults=DEFAULT_CONSTANTS):
self = cls()
self.connection = await aiosqlite.connect(":memory:")
coin_store = await CoinStore.create(DBWrapper(self.connection))
self.mempool_manager = MempoolManager(coin_store, defaults)
self.block_records = []
self.blocks = []
self.timestamp = 1
self.block_height = 0
self.defaults = defaults
return self
async def create_blockchain(constants: ConsensusConstants, db_version: int):
global blockchain_db_counter
db_path = Path(f"blockchain_test-{blockchain_db_counter}.db")
if db_path.exists():
db_path.unlink()
blockchain_db_counter += 1
connection = await aiosqlite.connect(db_path)
wrapper = DBWrapper(connection, False, db_version)
coin_store = await CoinStore.create(wrapper)
store = await BlockStore.create(wrapper)
hint_store = await HintStore.create(wrapper)
bc1 = await Blockchain.create(coin_store, store, constants, hint_store, Path("."))
assert bc1.get_peak() is None
return bc1, connection, db_path
async def test_store(self):
db_filename = Path("wallet_store_test.db")
if db_filename.exists():
db_filename.unlink()
db_connection = await aiosqlite.connect(db_filename)
db_wrapper = DBWrapper(db_connection)
store = await WalletInterestedStore.create(db_wrapper)
try:
coin_1 = Coin(token_bytes(32), token_bytes(32), uint64(12312))
coin_2 = Coin(token_bytes(32), token_bytes(32), uint64(12312))
assert (await store.get_interested_coin_ids()) == []
await store.add_interested_coin_id(coin_1.name())
assert (await store.get_interested_coin_ids()) == [coin_1.name()]
await store.add_interested_coin_id(coin_1.name())
assert (await store.get_interested_coin_ids()) == [coin_1.name()]
await store.add_interested_coin_id(coin_2.name())
assert set(await store.get_interested_coin_ids()) == {
coin_1.name(), coin_2.name()
}
puzzle_hash = token_bytes(32)
assert len(await store.get_interested_puzzle_hashes()) == 0
await store.add_interested_puzzle_hash(puzzle_hash, 2)
assert len(await store.get_interested_puzzle_hashes()) == 1
await store.add_interested_puzzle_hash(puzzle_hash, 2)
assert len(await store.get_interested_puzzle_hashes()) == 1
assert (
await
store.get_interested_puzzle_hash_wallet_id(puzzle_hash)) == 2
await store.add_interested_puzzle_hash(puzzle_hash, 3)
assert len(await store.get_interested_puzzle_hashes()) == 1
assert (
await
store.get_interested_puzzle_hash_wallet_id(puzzle_hash)) == 3
await store.remove_interested_puzzle_hash(puzzle_hash)
assert (await store.get_interested_puzzle_hash_wallet_id(
puzzle_hash)) is None
assert len(await store.get_interested_puzzle_hashes()) == 0
finally:
await db_connection.close()
db_filename.unlink()
async def test_store(self):
db_filename = Path("wallet_store_test.db")
if db_filename.exists():
db_filename.unlink()
db_connection = await aiosqlite.connect(db_filename)
db_wrapper = DBWrapper(db_connection)
store = await KeyValStore.create(db_wrapper)
try:
blocks = bt.get_consecutive_blocks(20)
block: FullBlock = blocks[0]
block_2: FullBlock = blocks[1]
assert (await store.get_object("a", FullBlock)) is None
await store.set_object("a", block)
assert await store.get_object("a", FullBlock) == block
await store.set_object("a", block)
assert await store.get_object("a", FullBlock) == block
await store.set_object("a", block_2)
await store.set_object("a", block_2)
assert await store.get_object("a", FullBlock) == block_2
await store.remove_object("a")
assert (await store.get_object("a", FullBlock)) is None
for block in blocks:
assert (await store.get_object(block.header_hash.hex(),
FullBlock)) is None
await store.set_object(block.header_hash.hex(), block)
assert (await store.get_object(block.header_hash.hex(),
FullBlock)) == block
# Wrong type
await store.set_object("a", block_2)
with pytest.raises(Exception):
await store.get_object("a", HeaderBlock)
finally:
await db_connection.close()
db_filename.unlink()
async def setup_db(name: str, db_version: int) -> DBWrapper:
db_filename = Path(name)
try:
os.unlink(db_filename)
except FileNotFoundError:
pass
connection = await aiosqlite.connect(db_filename)
def sql_trace_callback(req: str):
sql_log_path = "sql.log"
timestamp = datetime.now().strftime("%H:%M:%S.%f")
log = open(sql_log_path, "a")
log.write(timestamp + " " + req + "\n")
log.close()
if "--sql-logging" in sys.argv:
await connection.set_trace_callback(sql_trace_callback)
await connection.execute("pragma journal_mode=wal")
await connection.execute("pragma synchronous=full")
return DBWrapper(connection, False, db_version)
async def test_basic_coin_store(self):
wallet_a = WALLET_A
reward_ph = wallet_a.get_new_puzzlehash()
# Generate some coins
blocks = bt.get_consecutive_blocks(
10,
[],
farmer_reward_puzzle_hash=reward_ph,
pool_reward_puzzle_hash=reward_ph,
)
coins_to_spend: List[Coin] = []
for block in blocks:
if block.is_transaction_block():
for coin in block.get_included_reward_coins():
if coin.puzzle_hash == reward_ph:
coins_to_spend.append(coin)
spend_bundle = wallet_a.generate_signed_transaction(
1000, wallet_a.get_new_puzzlehash(), coins_to_spend[0])
db_path = Path("fndb_test.db")
if db_path.exists():
db_path.unlink()
connection = await aiosqlite.connect(db_path)
db_wrapper = DBWrapper(connection)
coin_store = await CoinStore.create(db_wrapper)
blocks = bt.get_consecutive_blocks(
10,
blocks,
farmer_reward_puzzle_hash=reward_ph,
pool_reward_puzzle_hash=reward_ph,
transaction_data=spend_bundle,
)
# Adding blocks to the coin store
should_be_included_prev: Set[Coin] = set()
should_be_included: Set[Coin] = set()
for block in blocks:
farmer_coin, pool_coin = get_future_reward_coins(block)
should_be_included.add(farmer_coin)
should_be_included.add(pool_coin)
if block.is_transaction_block():
removals, additions = run_and_get_removals_and_additions(
block, constants.MAX_BLOCK_COST_CLVM)
assert block.get_included_reward_coins(
) == should_be_included_prev
await coin_store.new_block(block, additions, removals)
for expected_coin in should_be_included_prev:
# Check that the coinbase rewards are added
record = await coin_store.get_coin_record(
expected_coin.name())
assert record is not None
assert not record.spent
assert record.coin == expected_coin
for coin_name in removals:
# Check that the removed coins are set to spent
record = await coin_store.get_coin_record(coin_name)
assert record.spent
for coin in additions:
# Check that the added coins are added
record = await coin_store.get_coin_record(coin.name())
assert not record.spent
assert coin == record.coin
should_be_included_prev = should_be_included.copy()
should_be_included = set()
await connection.close()
Path("fndb_test.db").unlink()
async def test_blocks(self):
blocks = bt.get_consecutive_blocks(758)
hints: List[Tuple[bytes32, bytes]] = []
for i in range(351):
hints.append((bytes32(rand_bytes(32)), rand_bytes(20)))
# the v1 schema allows duplicates in the hints table
for i in range(10):
coin_id = bytes32(rand_bytes(32))
hint = rand_bytes(20)
hints.append((coin_id, hint))
hints.append((coin_id, hint))
for i in range(2000):
hints.append((bytes32(rand_bytes(32)), rand_bytes(20)))
for i in range(5):
coin_id = bytes32(rand_bytes(32))
hint = rand_bytes(20)
hints.append((coin_id, hint))
hints.append((coin_id, hint))
with TempFile() as in_file, TempFile() as out_file:
async with aiosqlite.connect(in_file) as conn:
db_wrapper1 = DBWrapper(conn, 1)
block_store1 = await BlockStore.create(db_wrapper1)
coin_store1 = await CoinStore.create(db_wrapper1, 0)
hint_store1 = await HintStore.create(db_wrapper1)
for hint in hints:
await hint_store1.add_hints([(hint[0], hint[1])])
bc = await Blockchain.create(coin_store1,
block_store1,
test_constants,
hint_store1,
Path("."),
reserved_cores=0)
await db_wrapper1.commit_transaction()
for block in blocks:
await _validate_and_add_block(bc, block)
# now, convert v1 in_file to v2 out_file
await convert_v1_to_v2(in_file, out_file)
async with aiosqlite.connect(out_file) as conn2:
db_wrapper2 = DBWrapper(conn2, 2)
block_store2 = await BlockStore.create(db_wrapper2)
coin_store2 = await CoinStore.create(db_wrapper2, 0)
hint_store2 = await HintStore.create(db_wrapper2)
# check hints
for hint in hints:
assert hint[0] in await hint_store1.get_coin_ids(
hint[1])
assert hint[0] in await hint_store2.get_coin_ids(
hint[1])
# check peak
assert await block_store1.get_peak(
) == await block_store2.get_peak()
# check blocks
for block in blocks:
hh = block.header_hash
height = block.height
assert await block_store1.get_full_block(
hh) == await block_store2.get_full_block(hh)
assert await block_store1.get_full_block_bytes(
hh) == await block_store2.get_full_block_bytes(hh)
assert await block_store1.get_full_blocks_at([
height
]) == await block_store2.get_full_blocks_at([height])
assert await block_store1.get_block_records_by_hash(
[hh]
) == await block_store2.get_block_records_by_hash([hh])
assert await block_store1.get_block_record(
hh) == await block_store2.get_block_record(hh)
assert await block_store1.is_fully_compactified(
hh) == await block_store2.is_fully_compactified(hh)
# check coins
for block in blocks:
coins = await coin_store1.get_coins_added_at_height(
block.height)
assert await coin_store2.get_coins_added_at_height(
block.height) == coins
assert await coin_store1.get_coins_removed_at_height(
block.height
) == await coin_store2.get_coins_removed_at_height(
block.height)
for c in coins:
n = c.coin.name()
assert await coin_store1.get_coin_record(
n) == await coin_store2.get_coin_record(n)
async def create(
private_key: PrivateKey,
config: Dict,
db_path: Path,
constants: ConsensusConstants,
server: ChiaServer,
name: str = None,
):
self = WalletStateManager()
self.new_wallet = False
self.config = config
self.constants = constants
self.server = server
if name:
self.log = logging.getLogger(name)
else:
self.log = logging.getLogger(__name__)
self.lock = asyncio.Lock()
self.log.debug(f"Starting in db path: {db_path}")
self.db_connection = await aiosqlite.connect(db_path)
self.db_wrapper = DBWrapper(self.db_connection)
self.coin_store = await WalletCoinStore.create(self.db_wrapper)
self.tx_store = await WalletTransactionStore.create(self.db_wrapper)
self.puzzle_store = await WalletPuzzleStore.create(self.db_wrapper)
self.user_store = await WalletUserStore.create(self.db_wrapper)
self.action_store = await WalletActionStore.create(self.db_wrapper)
self.basic_store = await KeyValStore.create(self.db_wrapper)
self.trade_manager = await TradeManager.create(self, self.db_wrapper)
self.user_settings = await UserSettings.create(self.basic_store)
self.block_store = await WalletBlockStore.create(self.db_wrapper)
self.blockchain = await WalletBlockchain.create(
self.block_store,
self.constants,
self.coins_of_interest_received,
self.reorg_rollback,
)
self.weight_proof_handler = WeightProofHandler(self.constants,
self.blockchain)
self.sync_mode = False
self.sync_store = await WalletSyncStore.create()
self.state_changed_callback = None
self.pending_tx_callback = None
self.db_path = db_path
main_wallet_info = await self.user_store.get_wallet_by_id(1)
assert main_wallet_info is not None
self.private_key = private_key
self.main_wallet = await Wallet.create(self, main_wallet_info)
self.wallets = {main_wallet_info.id: self.main_wallet}
wallet = None
for wallet_info in await self.get_all_wallet_info_entries():
# self.log.info(f"wallet_info {wallet_info}")
if wallet_info.type == WalletType.STANDARD_WALLET:
if wallet_info.id == 1:
continue
wallet = await Wallet.create(config, wallet_info)
elif wallet_info.type == WalletType.COLOURED_COIN:
wallet = await CCWallet.create(
self,
self.main_wallet,
wallet_info,
)
elif wallet_info.type == WalletType.RATE_LIMITED:
wallet = await RLWallet.create(self, wallet_info)
elif wallet_info.type == WalletType.DISTRIBUTED_ID:
wallet = await DIDWallet.create(
self,
self.main_wallet,
wallet_info,
)
if wallet is not None:
self.wallets[wallet_info.id] = wallet
async with self.puzzle_store.lock:
index = await self.puzzle_store.get_last_derivation_path()
if index is None or index < self.config[
"initial_num_public_keys"] - 1:
await self.create_more_puzzle_hashes(from_zero=True)
return self
async def test_basic_reorg(self):
initial_block_count = 30
reorg_length = 15
blocks = bt.get_consecutive_blocks(initial_block_count)
db_path = Path("blockchain_test.db")
if db_path.exists():
db_path.unlink()
connection = await aiosqlite.connect(db_path)
db_wrapper = DBWrapper(connection)
coin_store = await CoinStore.create(db_wrapper)
store = await BlockStore.create(db_wrapper)
b: Blockchain = await Blockchain.create(coin_store, store,
test_constants)
try:
for block in blocks:
await b.receive_block(block)
assert b.get_peak().height == initial_block_count - 1
for c, block in enumerate(blocks):
if block.is_transaction_block():
coins = block.get_included_reward_coins()
records: List[Optional[CoinRecord]] = [
await coin_store.get_coin_record(coin.name())
for coin in coins
]
for record in records:
assert not record.spent
assert record.confirmed_block_index == block.height
assert record.spent_block_index == 0
blocks_reorg_chain = bt.get_consecutive_blocks(
reorg_length, blocks[:initial_block_count - 10], seed=b"2")
for reorg_block in blocks_reorg_chain:
result, error_code, _ = await b.receive_block(reorg_block)
print(
f"Height {reorg_block.height} {initial_block_count - 10} result {result}"
)
if reorg_block.height < initial_block_count - 10:
assert result == ReceiveBlockResult.ALREADY_HAVE_BLOCK
elif reorg_block.height < initial_block_count - 1:
assert result == ReceiveBlockResult.ADDED_AS_ORPHAN
elif reorg_block.height >= initial_block_count:
assert result == ReceiveBlockResult.NEW_PEAK
if reorg_block.is_transaction_block():
coins = reorg_block.get_included_reward_coins()
records: List[Optional[CoinRecord]] = [
await coin_store.get_coin_record(coin.name())
for coin in coins
]
for record in records:
assert not record.spent
assert record.confirmed_block_index == reorg_block.height
assert record.spent_block_index == 0
assert error_code is None
assert b.get_peak(
).height == initial_block_count - 10 + reorg_length - 1
except Exception as e:
await connection.close()
Path("blockchain_test.db").unlink()
b.shut_down()
raise e
await connection.close()
Path("blockchain_test.db").unlink()
b.shut_down()
async def test_basic_coin_store(self, rust_checker: bool):
wallet_a = WALLET_A
reward_ph = wallet_a.get_new_puzzlehash()
for cache_size in [0]:
# Generate some coins
blocks = bt.get_consecutive_blocks(
10,
[],
farmer_reward_puzzle_hash=reward_ph,
pool_reward_puzzle_hash=reward_ph,
)
coins_to_spend: List[Coin] = []
for block in blocks:
if block.is_transaction_block():
for coin in block.get_included_reward_coins():
if coin.puzzle_hash == reward_ph:
coins_to_spend.append(coin)
spend_bundle = wallet_a.generate_signed_transaction(
uint64(1000), wallet_a.get_new_puzzlehash(), coins_to_spend[0])
db_path = Path("fndb_test.db")
if db_path.exists():
db_path.unlink()
connection = await aiosqlite.connect(db_path)
db_wrapper = DBWrapper(connection)
coin_store = await CoinStore.create(db_wrapper,
cache_size=uint32(cache_size))
blocks = bt.get_consecutive_blocks(
10,
blocks,
farmer_reward_puzzle_hash=reward_ph,
pool_reward_puzzle_hash=reward_ph,
transaction_data=spend_bundle,
)
# Adding blocks to the coin store
should_be_included_prev: Set[Coin] = set()
should_be_included: Set[Coin] = set()
for block in blocks:
farmer_coin, pool_coin = get_future_reward_coins(block)
should_be_included.add(farmer_coin)
should_be_included.add(pool_coin)
if block.is_transaction_block():
if block.transactions_generator is not None:
block_gen: BlockGenerator = BlockGenerator(
block.transactions_generator, [])
npc_result = get_name_puzzle_conditions(
block_gen,
bt.constants.MAX_BLOCK_COST_CLVM,
cost_per_byte=bt.constants.COST_PER_BYTE,
safe_mode=False,
rust_checker=rust_checker,
)
tx_removals, tx_additions = tx_removals_and_additions(
npc_result.npc_list)
else:
tx_removals, tx_additions = [], []
assert block.get_included_reward_coins(
) == should_be_included_prev
await coin_store.new_block(block, tx_additions,
tx_removals)
if block.height != 0:
with pytest.raises(Exception):
await coin_store.new_block(block, tx_additions,
tx_removals)
for expected_coin in should_be_included_prev:
# Check that the coinbase rewards are added
record = await coin_store.get_coin_record(
expected_coin.name())
assert record is not None
assert not record.spent
assert record.coin == expected_coin
for coin_name in tx_removals:
# Check that the removed coins are set to spent
record = await coin_store.get_coin_record(coin_name)
assert record.spent
for coin in tx_additions:
# Check that the added coins are added
record = await coin_store.get_coin_record(coin.name())
assert not record.spent
assert coin == record.coin
should_be_included_prev = should_be_included.copy()
should_be_included = set()
await connection.close()
Path("fndb_test.db").unlink()
async def __aenter__(self) -> DBWrapper:
self.db_path = Path(tempfile.NamedTemporaryFile().name)
if self.db_path.exists():
self.db_path.unlink()
self.connection = await aiosqlite.connect(self.db_path)
return DBWrapper(self.connection, False, self.db_version)
async def test_puzzle_store(self):
db_filename = Path("puzzle_store_test.db")
if db_filename.exists():
db_filename.unlink()
con = await aiosqlite.connect(db_filename)
wrapper = DBWrapper(con)
db = await WalletPuzzleStore.create(wrapper)
try:
derivation_recs = []
# wallet_types = [t for t in WalletType]
[t for t in WalletType]
for i in range(1000):
derivation_recs.append(
DerivationRecord(
uint32(i),
token_bytes(32),
AugSchemeMPL.key_gen(token_bytes(32)).get_g1(),
WalletType.STANDARD_WALLET,
uint32(1),
False,
)
)
derivation_recs.append(
DerivationRecord(
uint32(i),
token_bytes(32),
AugSchemeMPL.key_gen(token_bytes(32)).get_g1(),
WalletType.RATE_LIMITED,
uint32(2),
False,
)
)
assert await db.puzzle_hash_exists(derivation_recs[0].puzzle_hash) is False
assert await db.index_for_pubkey(derivation_recs[0].pubkey) is None
assert await db.index_for_puzzle_hash(derivation_recs[2].puzzle_hash) is None
assert await db.wallet_info_for_puzzle_hash(derivation_recs[2].puzzle_hash) is None
assert len((await db.get_all_puzzle_hashes())) == 0
assert await db.get_last_derivation_path() is None
assert await db.get_unused_derivation_path() is None
assert await db.get_derivation_record(0, 2, False) is None
await db.add_derivation_paths(derivation_recs)
assert await db.puzzle_hash_exists(derivation_recs[0].puzzle_hash) is True
phs_1 = [derivation_recs[0].puzzle_hash]
phs_2 = [32 * bytes([1]), derivation_recs[0].puzzle_hash]
phs_3 = [derivation_recs[0].puzzle_hash, 32 * bytes([1])]
phs_4 = [32 * bytes([1]), 32 * bytes([2])]
phs_5 = []
assert await db.one_of_puzzle_hashes_exists(phs_1) is True
assert await db.one_of_puzzle_hashes_exists(phs_2) is True
assert await db.one_of_puzzle_hashes_exists(phs_3) is True
assert await db.one_of_puzzle_hashes_exists(phs_4) is False
assert await db.one_of_puzzle_hashes_exists(phs_5) is False
assert await db.index_for_pubkey(derivation_recs[4].pubkey) == 2
assert await db.index_for_puzzle_hash(derivation_recs[2].puzzle_hash) == 1
assert await db.wallet_info_for_puzzle_hash(derivation_recs[2].puzzle_hash) == (
derivation_recs[2].wallet_id,
derivation_recs[2].wallet_type,
)
assert len((await db.get_all_puzzle_hashes())) == 2000
assert await db.get_last_derivation_path() == 999
assert await db.get_unused_derivation_path() == 0
assert await db.get_derivation_record(0, 2, False) == derivation_recs[1]
# Indeces up to 250
await db.set_used_up_to(249)
assert await db.get_unused_derivation_path() == 250
except Exception as e:
print(e, type(e))
await db._clear_database()
await db.close()
db_filename.unlink()
raise e
await db._clear_database()
await db.close()
db_filename.unlink()
async def convert_v1_to_v2(in_path: Path, out_path: Path) -> None:
import aiosqlite
from chia.util.db_wrapper import DBWrapper
if out_path.exists():
print(f"output file already exists. {out_path}")
raise RuntimeError("already exists")
print(f"opening file for reading: {in_path}")
async with aiosqlite.connect(in_path) as in_db:
try:
async with in_db.execute(
"SELECT * from database_version") as cursor:
row = await cursor.fetchone()
if row is not None and row[0] != 1:
print(
f"blockchain database already version {row[0]}\nDone")
raise RuntimeError("already v2")
except aiosqlite.OperationalError:
pass
store_v1 = await BlockStore.create(DBWrapper(in_db, db_version=1))
print(f"opening file for writing: {out_path}")
async with aiosqlite.connect(out_path) as out_db:
await out_db.execute("pragma journal_mode=OFF")
await out_db.execute("pragma synchronous=OFF")
await out_db.execute("pragma cache_size=131072")
await out_db.execute("pragma locking_mode=exclusive")
print("initializing v2 version")
await out_db.execute("CREATE TABLE database_version(version int)")
await out_db.execute("INSERT INTO database_version VALUES(?)",
(2, ))
print("initializing v2 block store")
await out_db.execute("CREATE TABLE full_blocks("
"header_hash blob PRIMARY KEY,"
"prev_hash blob,"
"height bigint,"
"sub_epoch_summary blob,"
"is_fully_compactified tinyint,"
"in_main_chain tinyint,"
"block blob,"
"block_record blob)")
await out_db.execute("CREATE TABLE sub_epoch_segments_v3("
"ses_block_hash blob PRIMARY KEY,"
"challenge_segments blob)")
await out_db.execute(
"CREATE TABLE current_peak(key int PRIMARY KEY, hash blob)")
peak_hash, peak_height = await store_v1.get_peak()
print(f"peak: {peak_hash.hex()} height: {peak_height}")
await out_db.execute("INSERT INTO current_peak VALUES(?, ?)",
(0, peak_hash))
await out_db.commit()
print("[1/5] converting full_blocks")
height = peak_height + 1
hh = peak_hash
commit_in = BLOCK_COMMIT_RATE
rate = 1.0
start_time = time()
block_start_time = start_time
block_values = []
async with in_db.execute(
"SELECT header_hash, prev_hash, block, sub_epoch_summary FROM block_records ORDER BY height DESC"
) as cursor:
async with in_db.execute(
"SELECT header_hash, height, is_fully_compactified, block FROM full_blocks ORDER BY height DESC"
) as cursor_2:
await out_db.execute("begin transaction")
async for row in cursor:
header_hash = bytes.fromhex(row[0])
if header_hash != hh:
continue
# progress cursor_2 until we find the header hash
while True:
row_2 = await cursor_2.fetchone()
if row_2 is None:
print(
f"ERROR: could not find block {hh.hex()}")
raise RuntimeError(
f"block {hh.hex()} not found")
if bytes.fromhex(row_2[0]) == hh:
break
assert row_2[1] == height - 1
height = row_2[1]
is_fully_compactified = row_2[2]
block_bytes = row_2[3]
prev_hash = bytes.fromhex(row[1])
block_record = row[2]
ses = row[3]
block_values.append((
hh,
prev_hash,
height,
ses,
is_fully_compactified,
1, # in_main_chain
zstd.compress(block_bytes),
block_record,
))
hh = prev_hash
if (height % 1000) == 0:
print(
f"\r{height: 10d} {(peak_height-height)*100/peak_height:.2f}% "
f"{rate:0.1f} blocks/s ETA: {height//rate} s ",
end="",
)
sys.stdout.flush()
commit_in -= 1
if commit_in == 0:
commit_in = BLOCK_COMMIT_RATE
await out_db.executemany(
"INSERT OR REPLACE INTO full_blocks VALUES(?, ?, ?, ?, ?, ?, ?, ?)",
block_values)
await out_db.commit()
await out_db.execute("begin transaction")
block_values = []
end_time = time()
rate = BLOCK_COMMIT_RATE / (end_time - start_time)
start_time = end_time
await out_db.executemany(
"INSERT OR REPLACE INTO full_blocks VALUES(?, ?, ?, ?, ?, ?, ?, ?)",
block_values)
await out_db.commit()
end_time = time()
print(
f"\r {end_time - block_start_time:.2f} seconds "
)
print("[2/5] converting sub_epoch_segments_v3")
commit_in = SES_COMMIT_RATE
ses_values = []
ses_start_time = time()
async with in_db.execute(
"SELECT ses_block_hash, challenge_segments FROM sub_epoch_segments_v3"
) as cursor:
count = 0
await out_db.execute("begin transaction")
async for row in cursor:
block_hash = bytes32.fromhex(row[0])
ses = row[1]
ses_values.append((block_hash, ses))
count += 1
if (count % 100) == 0:
print(f"\r{count:10d} ", end="")
sys.stdout.flush()
commit_in -= 1
if commit_in == 0:
commit_in = SES_COMMIT_RATE
await out_db.executemany(
"INSERT INTO sub_epoch_segments_v3 VALUES (?, ?)",
ses_values)
await out_db.commit()
await out_db.execute("begin transaction")
ses_values = []
await out_db.executemany(
"INSERT INTO sub_epoch_segments_v3 VALUES (?, ?)", ses_values)
await out_db.commit()
end_time = time()
print(
f"\r {end_time - ses_start_time:.2f} seconds "
)
print("[3/5] converting hint_store")
commit_in = HINT_COMMIT_RATE
hint_start_time = time()
hint_values = []
await out_db.execute(
"CREATE TABLE hints(coin_id blob, hint blob, UNIQUE (coin_id, hint))"
)
await out_db.commit()
async with in_db.execute(
"SELECT coin_id, hint FROM hints") as cursor:
count = 0
await out_db.execute("begin transaction")
async for row in cursor:
hint_values.append((row[0], row[1]))
commit_in -= 1
if commit_in == 0:
commit_in = HINT_COMMIT_RATE
await out_db.executemany(
"INSERT OR IGNORE INTO hints VALUES(?, ?) ON CONFLICT DO NOTHING",
hint_values)
await out_db.commit()
await out_db.execute("begin transaction")
hint_values = []
await out_db.executemany(
"INSERT OR IGNORE INTO hints VALUES (?, ?)", hint_values)
await out_db.commit()
end_time = time()
print(
f"\r {end_time - hint_start_time:.2f} seconds "
)
print("[4/5] converting coin_store")
await out_db.execute(
"CREATE TABLE coin_record("
"coin_name blob PRIMARY KEY,"
" confirmed_index bigint,"
" spent_index bigint," # if this is zero, it means the coin has not been spent
" coinbase int,"
" puzzle_hash blob,"
" coin_parent blob,"
" amount blob," # we use a blob of 8 bytes to store uint64
" timestamp bigint)")
await out_db.commit()
commit_in = COIN_COMMIT_RATE
rate = 1.0
start_time = time()
coin_values = []
coin_start_time = start_time
async with in_db.execute(
"SELECT coin_name, confirmed_index, spent_index, coinbase, puzzle_hash, coin_parent, amount, timestamp "
"FROM coin_record WHERE confirmed_index <= ?",
(peak_height, ),
) as cursor:
count = 0
await out_db.execute("begin transaction")
async for row in cursor:
spent_index = row[2]
# in order to convert a consistent snapshot of the
# blockchain state, any coin that was spent *after* our
# cutoff must be converted into an unspent coin
if spent_index > peak_height:
spent_index = 0
coin_values.append((
bytes.fromhex(row[0]),
row[1],
spent_index,
row[3],
bytes.fromhex(row[4]),
bytes.fromhex(row[5]),
row[6],
row[7],
))
count += 1
if (count % 2000) == 0:
print(
f"\r{count//1000:10d}k coins {rate:0.1f} coins/s ",
end="")
sys.stdout.flush()
commit_in -= 1
if commit_in == 0:
commit_in = COIN_COMMIT_RATE
await out_db.executemany(
"INSERT INTO coin_record VALUES(?, ?, ?, ?, ?, ?, ?, ?)",
coin_values)
await out_db.commit()
await out_db.execute("begin transaction")
coin_values = []
end_time = time()
rate = COIN_COMMIT_RATE / (end_time - start_time)
start_time = end_time
await out_db.executemany(
"INSERT INTO coin_record VALUES(?, ?, ?, ?, ?, ?, ?, ?)",
coin_values)
await out_db.commit()
end_time = time()
print(
f"\r {end_time - coin_start_time:.2f} seconds "
)
print("[5/5] build indices")
index_start_time = time()
print(" block store")
await BlockStore.create(DBWrapper(out_db, db_version=2))
print(" coin store")
await CoinStore.create(DBWrapper(out_db, db_version=2))
print(" hint store")
await HintStore.create(DBWrapper(out_db, db_version=2))
end_time = time()
print(
f"\r {end_time - index_start_time:.2f} seconds "
)
async def test_store(self):
db_filename = Path("wallet_store_test.db")
if db_filename.exists():
db_filename.unlink()
db_connection = await aiosqlite.connect(db_filename)
db_wrapper = DBWrapper(db_connection)
store = await WalletPoolStore.create(db_wrapper)
try:
await db_wrapper.begin_transaction()
coin_0 = Coin(token_bytes(32), token_bytes(32), uint64(12312))
coin_0_alt = Coin(token_bytes(32), token_bytes(32), uint64(12312))
solution_0: CoinSpend = make_child_solution(None, coin_0)
solution_0_alt: CoinSpend = make_child_solution(None, coin_0_alt)
solution_1: CoinSpend = make_child_solution(solution_0)
assert store.get_spends_for_wallet(0) == []
assert store.get_spends_for_wallet(1) == []
await store.add_spend(1, solution_1, 100)
assert store.get_spends_for_wallet(1) == [(100, solution_1)]
# Idempotent
await store.add_spend(1, solution_1, 100)
assert store.get_spends_for_wallet(1) == [(100, solution_1)]
with pytest.raises(ValueError):
await store.add_spend(1, solution_1, 101)
# Rebuild cache, no longer present
await db_wrapper.rollback_transaction()
await store.rebuild_cache()
assert store.get_spends_for_wallet(1) == []
await store.rebuild_cache()
await store.add_spend(1, solution_1, 100)
assert store.get_spends_for_wallet(1) == [(100, solution_1)]
solution_1_alt: CoinSpend = make_child_solution(solution_0_alt)
with pytest.raises(ValueError):
await store.add_spend(1, solution_1_alt, 100)
assert store.get_spends_for_wallet(1) == [(100, solution_1)]
solution_2: CoinSpend = make_child_solution(solution_1)
await store.add_spend(1, solution_2, 100)
await store.rebuild_cache()
solution_3: CoinSpend = make_child_solution(solution_2)
await store.add_spend(1, solution_3, 100)
solution_4: CoinSpend = make_child_solution(solution_3)
with pytest.raises(ValueError):
await store.add_spend(1, solution_4, 99)
await store.rebuild_cache()
await store.add_spend(1, solution_4, 101)
await store.rebuild_cache()
await store.rollback(101, 1)
await store.rebuild_cache()
assert store.get_spends_for_wallet(1) == [
(100, solution_1),
(100, solution_2),
(100, solution_3),
(101, solution_4),
]
await store.rebuild_cache()
await store.rollback(100, 1)
await store.rebuild_cache()
assert store.get_spends_for_wallet(1) == [
(100, solution_1),
(100, solution_2),
(100, solution_3),
]
with pytest.raises(ValueError):
await store.add_spend(1, solution_1, 105)
await store.add_spend(1, solution_4, 105)
solution_5: CoinSpend = make_child_solution(solution_4)
await store.add_spend(1, solution_5, 105)
await store.rollback(99, 1)
assert store.get_spends_for_wallet(1) == []
finally:
await db_connection.close()
db_filename.unlink()