async def test_block_store(self): assert sqlite3.threadsafety == 1 blocks = bt.get_consecutive_blocks(10) db_filename = Path("blockchain_test.db") db_filename_2 = Path("blockchain_test2.db") if db_filename.exists(): db_filename.unlink() if db_filename_2.exists(): db_filename_2.unlink() connection = await aiosqlite.connect(db_filename) connection_2 = await aiosqlite.connect(db_filename_2) db_wrapper = DBWrapper(connection) db_wrapper_2 = DBWrapper(connection_2) # Use a different file for the blockchain coin_store_2 = await CoinStore.create(db_wrapper_2) store_2 = await BlockStore.create(db_wrapper_2) bc = await Blockchain.create(coin_store_2, store_2, test_constants) store = await BlockStore.create(db_wrapper) await BlockStore.create(db_wrapper_2) try: # Save/get block for block in blocks: await bc.receive_block(block) block_record = bc.block_record(block.header_hash) block_record_hh = block_record.header_hash await store.add_full_block(block.header_hash, block, block_record) await store.add_full_block(block.header_hash, block, block_record) assert block == await store.get_full_block(block.header_hash) assert block == await store.get_full_block(block.header_hash) assert block_record == (await store.get_block_record(block_record_hh)) await store.set_peak(block_record.header_hash) await store.set_peak(block_record.header_hash) assert len(await store.get_full_blocks_at([1])) == 1 assert len(await store.get_full_blocks_at([0])) == 1 assert len(await store.get_full_blocks_at([100])) == 0 # Get blocks block_record_records = await store.get_block_records() assert len(block_record_records[0]) == len(blocks) # Peak is correct assert block_record_records[1] == blocks[-1].header_hash except Exception: await connection.close() await connection_2.close() db_filename.unlink() db_filename_2.unlink() raise await connection.close() await connection_2.close() db_filename.unlink() db_filename_2.unlink()
async def test_set_spent(self): blocks = bt.get_consecutive_blocks(9, []) db_path = Path("fndb_test.db") if db_path.exists(): db_path.unlink() connection = await aiosqlite.connect(db_path) db_wrapper = DBWrapper(connection) coin_store = await CoinStore.create(db_wrapper) # Save/get block for block in blocks: if block.is_transaction_block(): await coin_store.new_block(block) coins = block.get_included_reward_coins() records = [ await coin_store.get_coin_record(coin.name()) for coin in coins ] for record in records: await coin_store._set_spent(record.coin.name(), block.height) records = [ await coin_store.get_coin_record(coin.name()) for coin in coins ] for record in records: assert record.spent assert record.spent_block_index == block.height await connection.close() Path("fndb_test.db").unlink()
async def test_get_puzzle_hash(self): for cache_size in [0, 10, 100000]: num_blocks = 20 farmer_ph = 32 * b"0" pool_ph = 32 * b"1" blocks = bt.get_consecutive_blocks( num_blocks, farmer_reward_puzzle_hash=farmer_ph, pool_reward_puzzle_hash=pool_ph, guarantee_transaction_block=True, ) db_path = Path("blockchain_test.db") if db_path.exists(): db_path.unlink() connection = await aiosqlite.connect(db_path) db_wrapper = DBWrapper(connection) coin_store = await CoinStore.create(db_wrapper, cache_size=uint32(cache_size)) store = await BlockStore.create(db_wrapper) b: Blockchain = await Blockchain.create(coin_store, store, test_constants) for block in blocks: res, err, _ = await b.receive_block(block) assert err is None assert res == ReceiveBlockResult.NEW_PEAK assert b.get_peak().height == num_blocks - 1 coins_farmer = await coin_store.get_coin_records_by_puzzle_hash(True, pool_ph) coins_pool = await coin_store.get_coin_records_by_puzzle_hash(True, farmer_ph) assert len(coins_farmer) == num_blocks - 2 assert len(coins_pool) == num_blocks - 2 await connection.close() Path("blockchain_test.db").unlink() b.shut_down()
async def test_store(): db_filename = Path("wallet_user_store_test.db") if db_filename.exists(): db_filename.unlink() db_connection = await aiosqlite.connect(db_filename) db_wrapper = DBWrapper(db_connection) store = await WalletUserStore.create(db_wrapper) try: await store.init_wallet() wallet = None for i in range(1, 5): assert (await store.get_last_wallet()).id == i wallet = await store.create_wallet("CAT_WALLET", WalletType.CAT, "abc") assert wallet.id == i + 1 assert wallet.id == 5 for i in range(2, 6): await store.delete_wallet(i, in_transaction=False) assert (await store.get_last_wallet()).id == 1 wallet = await store.create_wallet("CAT_WALLET", WalletType.CAT, "abc") # Due to autoincrement, we don't reuse IDs assert (await store.get_last_wallet()).id == 6 assert wallet.id == 6 assert (await store.get_wallet_by_id(7)) is None assert (await store.get_wallet_by_id(6)) == wallet assert await store.get_last_wallet() == wallet finally: await db_connection.close() db_filename.unlink()
async def test_set_spent(self): blocks = bt.get_consecutive_blocks(9, []) for cache_size in [0, 10, 100000]: db_path = Path("fndb_test.db") if db_path.exists(): db_path.unlink() connection = await aiosqlite.connect(db_path) db_wrapper = DBWrapper(connection) coin_store = await CoinStore.create(db_wrapper, cache_size=uint32(cache_size)) # Save/get block for block in blocks: if block.is_transaction_block(): removals, additions = [], [] await coin_store.new_block(block, additions, removals) coins = block.get_included_reward_coins() records = [await coin_store.get_coin_record(coin.name()) for coin in coins] for record in records: await coin_store._set_spent(record.coin.name(), block.height) with pytest.raises(AssertionError): await coin_store._set_spent(record.coin.name(), block.height) records = [await coin_store.get_coin_record(coin.name()) for coin in coins] for record in records: assert record.spent assert record.spent_block_index == block.height await connection.close() Path("fndb_test.db").unlink()
async def test_deadlock(self): """ This test was added because the store was deadlocking in certain situations, when fetching and adding blocks repeatedly. The issue was patched. """ blocks = bt.get_consecutive_blocks(10) db_filename = Path("blockchain_test.db") db_filename_2 = Path("blockchain_test2.db") if db_filename.exists(): db_filename.unlink() if db_filename_2.exists(): db_filename_2.unlink() connection = await aiosqlite.connect(db_filename) connection_2 = await aiosqlite.connect(db_filename_2) wrapper = DBWrapper(connection) wrapper_2 = DBWrapper(connection_2) store = await BlockStore.create(wrapper) coin_store_2 = await CoinStore.create(wrapper_2) store_2 = await BlockStore.create(wrapper_2) bc = await Blockchain.create(coin_store_2, store_2, test_constants) block_records = [] for block in blocks: await bc.receive_block(block) block_records.append(bc.block_record(block.header_hash)) tasks = [] for i in range(10000): rand_i = random.randint(0, 9) if random.random() < 0.5: tasks.append( asyncio.create_task( store.add_full_block(blocks[rand_i].header_hash, blocks[rand_i], block_records[rand_i]))) if random.random() < 0.5: tasks.append( asyncio.create_task( store.get_full_block(blocks[rand_i].header_hash))) await asyncio.gather(*tasks) await connection.close() await connection_2.close() db_filename.unlink() db_filename_2.unlink()
async def create_ram_blockchain( consensus_constants: ConsensusConstants ) -> Tuple[aiosqlite.Connection, Blockchain]: connection = await aiosqlite.connect(":memory:") db_wrapper = DBWrapper(connection) block_store = await BlockStore.create(db_wrapper) coin_store = await CoinStore.create(db_wrapper) blockchain = await Blockchain.create(coin_store, block_store, consensus_constants) return connection, blockchain
async def create(cls): self = cls() self.connection = await aiosqlite.connect(":memory:") coin_store = await CoinStore.create(DBWrapper(self.connection)) self.mempool_manager = MempoolManager(coin_store, DEFAULT_CONSTANTS) self.block_records = [] self.blocks = [] self.timestamp = DEFAULT_CONSTANTS.INITIAL_FREEZE_END_TIMESTAMP + 1 self.block_height = 0 return self
async def test_rollback(self): blocks = bt.get_consecutive_blocks(20) for cache_size in [0, 10, 100000]: db_path = Path("fndb_test.db") if db_path.exists(): db_path.unlink() connection = await aiosqlite.connect(db_path) db_wrapper = DBWrapper(connection) coin_store = await CoinStore.create(db_wrapper, cache_size=uint32(cache_size)) for block in blocks: if block.is_transaction_block(): removals, additions = [], [] await coin_store.new_block(block, additions, removals) coins = block.get_included_reward_coins() records: List[Optional[CoinRecord]] = [ await coin_store.get_coin_record(coin.name()) for coin in coins ] for record in records: await coin_store._set_spent(record.coin.name(), block.height) records: List[Optional[CoinRecord]] = [ await coin_store.get_coin_record(coin.name()) for coin in coins ] for record in records: assert record.spent assert record.spent_block_index == block.height reorg_index = 8 await coin_store.rollback_to_block(reorg_index) for block in blocks: if block.is_transaction_block(): coins = block.get_included_reward_coins() records: List[Optional[CoinRecord]] = [ await coin_store.get_coin_record(coin.name()) for coin in coins ] if block.height <= reorg_index: for record in records: assert record is not None assert record.spent else: for record in records: assert record is None await connection.close() Path("fndb_test.db").unlink()
async def create_blockchain(constants: ConsensusConstants): db_path = Path("blockchain_test.db") if db_path.exists(): db_path.unlink() connection = await aiosqlite.connect(db_path) wrapper = DBWrapper(connection) coin_store = await CoinStore.create(wrapper) store = await BlockStore.create(wrapper) bc1 = await Blockchain.create(coin_store, store, constants) assert bc1.get_peak() is None return bc1, connection, db_path
async def create(cls, defaults=DEFAULT_CONSTANTS): self = cls() self.connection = await aiosqlite.connect(":memory:") coin_store = await CoinStore.create(DBWrapper(self.connection)) self.mempool_manager = MempoolManager(coin_store, defaults) self.block_records = [] self.blocks = [] self.timestamp = 1 self.block_height = 0 self.defaults = defaults return self
async def create_blockchain(constants: ConsensusConstants, db_version: int): global blockchain_db_counter db_path = Path(f"blockchain_test-{blockchain_db_counter}.db") if db_path.exists(): db_path.unlink() blockchain_db_counter += 1 connection = await aiosqlite.connect(db_path) wrapper = DBWrapper(connection, False, db_version) coin_store = await CoinStore.create(wrapper) store = await BlockStore.create(wrapper) hint_store = await HintStore.create(wrapper) bc1 = await Blockchain.create(coin_store, store, constants, hint_store, Path(".")) assert bc1.get_peak() is None return bc1, connection, db_path
async def test_store(self): db_filename = Path("wallet_store_test.db") if db_filename.exists(): db_filename.unlink() db_connection = await aiosqlite.connect(db_filename) db_wrapper = DBWrapper(db_connection) store = await WalletInterestedStore.create(db_wrapper) try: coin_1 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) coin_2 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) assert (await store.get_interested_coin_ids()) == [] await store.add_interested_coin_id(coin_1.name()) assert (await store.get_interested_coin_ids()) == [coin_1.name()] await store.add_interested_coin_id(coin_1.name()) assert (await store.get_interested_coin_ids()) == [coin_1.name()] await store.add_interested_coin_id(coin_2.name()) assert set(await store.get_interested_coin_ids()) == { coin_1.name(), coin_2.name() } puzzle_hash = token_bytes(32) assert len(await store.get_interested_puzzle_hashes()) == 0 await store.add_interested_puzzle_hash(puzzle_hash, 2) assert len(await store.get_interested_puzzle_hashes()) == 1 await store.add_interested_puzzle_hash(puzzle_hash, 2) assert len(await store.get_interested_puzzle_hashes()) == 1 assert ( await store.get_interested_puzzle_hash_wallet_id(puzzle_hash)) == 2 await store.add_interested_puzzle_hash(puzzle_hash, 3) assert len(await store.get_interested_puzzle_hashes()) == 1 assert ( await store.get_interested_puzzle_hash_wallet_id(puzzle_hash)) == 3 await store.remove_interested_puzzle_hash(puzzle_hash) assert (await store.get_interested_puzzle_hash_wallet_id( puzzle_hash)) is None assert len(await store.get_interested_puzzle_hashes()) == 0 finally: await db_connection.close() db_filename.unlink()
async def test_store(self): db_filename = Path("wallet_store_test.db") if db_filename.exists(): db_filename.unlink() db_connection = await aiosqlite.connect(db_filename) db_wrapper = DBWrapper(db_connection) store = await KeyValStore.create(db_wrapper) try: blocks = bt.get_consecutive_blocks(20) block: FullBlock = blocks[0] block_2: FullBlock = blocks[1] assert (await store.get_object("a", FullBlock)) is None await store.set_object("a", block) assert await store.get_object("a", FullBlock) == block await store.set_object("a", block) assert await store.get_object("a", FullBlock) == block await store.set_object("a", block_2) await store.set_object("a", block_2) assert await store.get_object("a", FullBlock) == block_2 await store.remove_object("a") assert (await store.get_object("a", FullBlock)) is None for block in blocks: assert (await store.get_object(block.header_hash.hex(), FullBlock)) is None await store.set_object(block.header_hash.hex(), block) assert (await store.get_object(block.header_hash.hex(), FullBlock)) == block # Wrong type await store.set_object("a", block_2) with pytest.raises(Exception): await store.get_object("a", HeaderBlock) finally: await db_connection.close() db_filename.unlink()
async def setup_db(name: str, db_version: int) -> DBWrapper: db_filename = Path(name) try: os.unlink(db_filename) except FileNotFoundError: pass connection = await aiosqlite.connect(db_filename) def sql_trace_callback(req: str): sql_log_path = "sql.log" timestamp = datetime.now().strftime("%H:%M:%S.%f") log = open(sql_log_path, "a") log.write(timestamp + " " + req + "\n") log.close() if "--sql-logging" in sys.argv: await connection.set_trace_callback(sql_trace_callback) await connection.execute("pragma journal_mode=wal") await connection.execute("pragma synchronous=full") return DBWrapper(connection, False, db_version)
async def test_basic_coin_store(self): wallet_a = WALLET_A reward_ph = wallet_a.get_new_puzzlehash() # Generate some coins blocks = bt.get_consecutive_blocks( 10, [], farmer_reward_puzzle_hash=reward_ph, pool_reward_puzzle_hash=reward_ph, ) coins_to_spend: List[Coin] = [] for block in blocks: if block.is_transaction_block(): for coin in block.get_included_reward_coins(): if coin.puzzle_hash == reward_ph: coins_to_spend.append(coin) spend_bundle = wallet_a.generate_signed_transaction( 1000, wallet_a.get_new_puzzlehash(), coins_to_spend[0]) db_path = Path("fndb_test.db") if db_path.exists(): db_path.unlink() connection = await aiosqlite.connect(db_path) db_wrapper = DBWrapper(connection) coin_store = await CoinStore.create(db_wrapper) blocks = bt.get_consecutive_blocks( 10, blocks, farmer_reward_puzzle_hash=reward_ph, pool_reward_puzzle_hash=reward_ph, transaction_data=spend_bundle, ) # Adding blocks to the coin store should_be_included_prev: Set[Coin] = set() should_be_included: Set[Coin] = set() for block in blocks: farmer_coin, pool_coin = get_future_reward_coins(block) should_be_included.add(farmer_coin) should_be_included.add(pool_coin) if block.is_transaction_block(): removals, additions = run_and_get_removals_and_additions( block, constants.MAX_BLOCK_COST_CLVM) assert block.get_included_reward_coins( ) == should_be_included_prev await coin_store.new_block(block, additions, removals) for expected_coin in should_be_included_prev: # Check that the coinbase rewards are added record = await coin_store.get_coin_record( expected_coin.name()) assert record is not None assert not record.spent assert record.coin == expected_coin for coin_name in removals: # Check that the removed coins are set to spent record = await coin_store.get_coin_record(coin_name) assert record.spent for coin in additions: # Check that the added coins are added record = await coin_store.get_coin_record(coin.name()) assert not record.spent assert coin == record.coin should_be_included_prev = should_be_included.copy() should_be_included = set() await connection.close() Path("fndb_test.db").unlink()
async def test_blocks(self): blocks = bt.get_consecutive_blocks(758) hints: List[Tuple[bytes32, bytes]] = [] for i in range(351): hints.append((bytes32(rand_bytes(32)), rand_bytes(20))) # the v1 schema allows duplicates in the hints table for i in range(10): coin_id = bytes32(rand_bytes(32)) hint = rand_bytes(20) hints.append((coin_id, hint)) hints.append((coin_id, hint)) for i in range(2000): hints.append((bytes32(rand_bytes(32)), rand_bytes(20))) for i in range(5): coin_id = bytes32(rand_bytes(32)) hint = rand_bytes(20) hints.append((coin_id, hint)) hints.append((coin_id, hint)) with TempFile() as in_file, TempFile() as out_file: async with aiosqlite.connect(in_file) as conn: db_wrapper1 = DBWrapper(conn, 1) block_store1 = await BlockStore.create(db_wrapper1) coin_store1 = await CoinStore.create(db_wrapper1, 0) hint_store1 = await HintStore.create(db_wrapper1) for hint in hints: await hint_store1.add_hints([(hint[0], hint[1])]) bc = await Blockchain.create(coin_store1, block_store1, test_constants, hint_store1, Path("."), reserved_cores=0) await db_wrapper1.commit_transaction() for block in blocks: await _validate_and_add_block(bc, block) # now, convert v1 in_file to v2 out_file await convert_v1_to_v2(in_file, out_file) async with aiosqlite.connect(out_file) as conn2: db_wrapper2 = DBWrapper(conn2, 2) block_store2 = await BlockStore.create(db_wrapper2) coin_store2 = await CoinStore.create(db_wrapper2, 0) hint_store2 = await HintStore.create(db_wrapper2) # check hints for hint in hints: assert hint[0] in await hint_store1.get_coin_ids( hint[1]) assert hint[0] in await hint_store2.get_coin_ids( hint[1]) # check peak assert await block_store1.get_peak( ) == await block_store2.get_peak() # check blocks for block in blocks: hh = block.header_hash height = block.height assert await block_store1.get_full_block( hh) == await block_store2.get_full_block(hh) assert await block_store1.get_full_block_bytes( hh) == await block_store2.get_full_block_bytes(hh) assert await block_store1.get_full_blocks_at([ height ]) == await block_store2.get_full_blocks_at([height]) assert await block_store1.get_block_records_by_hash( [hh] ) == await block_store2.get_block_records_by_hash([hh]) assert await block_store1.get_block_record( hh) == await block_store2.get_block_record(hh) assert await block_store1.is_fully_compactified( hh) == await block_store2.is_fully_compactified(hh) # check coins for block in blocks: coins = await coin_store1.get_coins_added_at_height( block.height) assert await coin_store2.get_coins_added_at_height( block.height) == coins assert await coin_store1.get_coins_removed_at_height( block.height ) == await coin_store2.get_coins_removed_at_height( block.height) for c in coins: n = c.coin.name() assert await coin_store1.get_coin_record( n) == await coin_store2.get_coin_record(n)
async def create( private_key: PrivateKey, config: Dict, db_path: Path, constants: ConsensusConstants, server: ChiaServer, name: str = None, ): self = WalletStateManager() self.new_wallet = False self.config = config self.constants = constants self.server = server if name: self.log = logging.getLogger(name) else: self.log = logging.getLogger(__name__) self.lock = asyncio.Lock() self.log.debug(f"Starting in db path: {db_path}") self.db_connection = await aiosqlite.connect(db_path) self.db_wrapper = DBWrapper(self.db_connection) self.coin_store = await WalletCoinStore.create(self.db_wrapper) self.tx_store = await WalletTransactionStore.create(self.db_wrapper) self.puzzle_store = await WalletPuzzleStore.create(self.db_wrapper) self.user_store = await WalletUserStore.create(self.db_wrapper) self.action_store = await WalletActionStore.create(self.db_wrapper) self.basic_store = await KeyValStore.create(self.db_wrapper) self.trade_manager = await TradeManager.create(self, self.db_wrapper) self.user_settings = await UserSettings.create(self.basic_store) self.block_store = await WalletBlockStore.create(self.db_wrapper) self.blockchain = await WalletBlockchain.create( self.block_store, self.constants, self.coins_of_interest_received, self.reorg_rollback, ) self.weight_proof_handler = WeightProofHandler(self.constants, self.blockchain) self.sync_mode = False self.sync_store = await WalletSyncStore.create() self.state_changed_callback = None self.pending_tx_callback = None self.db_path = db_path main_wallet_info = await self.user_store.get_wallet_by_id(1) assert main_wallet_info is not None self.private_key = private_key self.main_wallet = await Wallet.create(self, main_wallet_info) self.wallets = {main_wallet_info.id: self.main_wallet} wallet = None for wallet_info in await self.get_all_wallet_info_entries(): # self.log.info(f"wallet_info {wallet_info}") if wallet_info.type == WalletType.STANDARD_WALLET: if wallet_info.id == 1: continue wallet = await Wallet.create(config, wallet_info) elif wallet_info.type == WalletType.COLOURED_COIN: wallet = await CCWallet.create( self, self.main_wallet, wallet_info, ) elif wallet_info.type == WalletType.RATE_LIMITED: wallet = await RLWallet.create(self, wallet_info) elif wallet_info.type == WalletType.DISTRIBUTED_ID: wallet = await DIDWallet.create( self, self.main_wallet, wallet_info, ) if wallet is not None: self.wallets[wallet_info.id] = wallet async with self.puzzle_store.lock: index = await self.puzzle_store.get_last_derivation_path() if index is None or index < self.config[ "initial_num_public_keys"] - 1: await self.create_more_puzzle_hashes(from_zero=True) return self
async def test_basic_reorg(self): initial_block_count = 30 reorg_length = 15 blocks = bt.get_consecutive_blocks(initial_block_count) db_path = Path("blockchain_test.db") if db_path.exists(): db_path.unlink() connection = await aiosqlite.connect(db_path) db_wrapper = DBWrapper(connection) coin_store = await CoinStore.create(db_wrapper) store = await BlockStore.create(db_wrapper) b: Blockchain = await Blockchain.create(coin_store, store, test_constants) try: for block in blocks: await b.receive_block(block) assert b.get_peak().height == initial_block_count - 1 for c, block in enumerate(blocks): if block.is_transaction_block(): coins = block.get_included_reward_coins() records: List[Optional[CoinRecord]] = [ await coin_store.get_coin_record(coin.name()) for coin in coins ] for record in records: assert not record.spent assert record.confirmed_block_index == block.height assert record.spent_block_index == 0 blocks_reorg_chain = bt.get_consecutive_blocks( reorg_length, blocks[:initial_block_count - 10], seed=b"2") for reorg_block in blocks_reorg_chain: result, error_code, _ = await b.receive_block(reorg_block) print( f"Height {reorg_block.height} {initial_block_count - 10} result {result}" ) if reorg_block.height < initial_block_count - 10: assert result == ReceiveBlockResult.ALREADY_HAVE_BLOCK elif reorg_block.height < initial_block_count - 1: assert result == ReceiveBlockResult.ADDED_AS_ORPHAN elif reorg_block.height >= initial_block_count: assert result == ReceiveBlockResult.NEW_PEAK if reorg_block.is_transaction_block(): coins = reorg_block.get_included_reward_coins() records: List[Optional[CoinRecord]] = [ await coin_store.get_coin_record(coin.name()) for coin in coins ] for record in records: assert not record.spent assert record.confirmed_block_index == reorg_block.height assert record.spent_block_index == 0 assert error_code is None assert b.get_peak( ).height == initial_block_count - 10 + reorg_length - 1 except Exception as e: await connection.close() Path("blockchain_test.db").unlink() b.shut_down() raise e await connection.close() Path("blockchain_test.db").unlink() b.shut_down()
async def test_basic_coin_store(self, rust_checker: bool): wallet_a = WALLET_A reward_ph = wallet_a.get_new_puzzlehash() for cache_size in [0]: # Generate some coins blocks = bt.get_consecutive_blocks( 10, [], farmer_reward_puzzle_hash=reward_ph, pool_reward_puzzle_hash=reward_ph, ) coins_to_spend: List[Coin] = [] for block in blocks: if block.is_transaction_block(): for coin in block.get_included_reward_coins(): if coin.puzzle_hash == reward_ph: coins_to_spend.append(coin) spend_bundle = wallet_a.generate_signed_transaction( uint64(1000), wallet_a.get_new_puzzlehash(), coins_to_spend[0]) db_path = Path("fndb_test.db") if db_path.exists(): db_path.unlink() connection = await aiosqlite.connect(db_path) db_wrapper = DBWrapper(connection) coin_store = await CoinStore.create(db_wrapper, cache_size=uint32(cache_size)) blocks = bt.get_consecutive_blocks( 10, blocks, farmer_reward_puzzle_hash=reward_ph, pool_reward_puzzle_hash=reward_ph, transaction_data=spend_bundle, ) # Adding blocks to the coin store should_be_included_prev: Set[Coin] = set() should_be_included: Set[Coin] = set() for block in blocks: farmer_coin, pool_coin = get_future_reward_coins(block) should_be_included.add(farmer_coin) should_be_included.add(pool_coin) if block.is_transaction_block(): if block.transactions_generator is not None: block_gen: BlockGenerator = BlockGenerator( block.transactions_generator, []) npc_result = get_name_puzzle_conditions( block_gen, bt.constants.MAX_BLOCK_COST_CLVM, cost_per_byte=bt.constants.COST_PER_BYTE, safe_mode=False, rust_checker=rust_checker, ) tx_removals, tx_additions = tx_removals_and_additions( npc_result.npc_list) else: tx_removals, tx_additions = [], [] assert block.get_included_reward_coins( ) == should_be_included_prev await coin_store.new_block(block, tx_additions, tx_removals) if block.height != 0: with pytest.raises(Exception): await coin_store.new_block(block, tx_additions, tx_removals) for expected_coin in should_be_included_prev: # Check that the coinbase rewards are added record = await coin_store.get_coin_record( expected_coin.name()) assert record is not None assert not record.spent assert record.coin == expected_coin for coin_name in tx_removals: # Check that the removed coins are set to spent record = await coin_store.get_coin_record(coin_name) assert record.spent for coin in tx_additions: # Check that the added coins are added record = await coin_store.get_coin_record(coin.name()) assert not record.spent assert coin == record.coin should_be_included_prev = should_be_included.copy() should_be_included = set() await connection.close() Path("fndb_test.db").unlink()
async def __aenter__(self) -> DBWrapper: self.db_path = Path(tempfile.NamedTemporaryFile().name) if self.db_path.exists(): self.db_path.unlink() self.connection = await aiosqlite.connect(self.db_path) return DBWrapper(self.connection, False, self.db_version)
async def test_puzzle_store(self): db_filename = Path("puzzle_store_test.db") if db_filename.exists(): db_filename.unlink() con = await aiosqlite.connect(db_filename) wrapper = DBWrapper(con) db = await WalletPuzzleStore.create(wrapper) try: derivation_recs = [] # wallet_types = [t for t in WalletType] [t for t in WalletType] for i in range(1000): derivation_recs.append( DerivationRecord( uint32(i), token_bytes(32), AugSchemeMPL.key_gen(token_bytes(32)).get_g1(), WalletType.STANDARD_WALLET, uint32(1), False, ) ) derivation_recs.append( DerivationRecord( uint32(i), token_bytes(32), AugSchemeMPL.key_gen(token_bytes(32)).get_g1(), WalletType.RATE_LIMITED, uint32(2), False, ) ) assert await db.puzzle_hash_exists(derivation_recs[0].puzzle_hash) is False assert await db.index_for_pubkey(derivation_recs[0].pubkey) is None assert await db.index_for_puzzle_hash(derivation_recs[2].puzzle_hash) is None assert await db.wallet_info_for_puzzle_hash(derivation_recs[2].puzzle_hash) is None assert len((await db.get_all_puzzle_hashes())) == 0 assert await db.get_last_derivation_path() is None assert await db.get_unused_derivation_path() is None assert await db.get_derivation_record(0, 2, False) is None await db.add_derivation_paths(derivation_recs) assert await db.puzzle_hash_exists(derivation_recs[0].puzzle_hash) is True phs_1 = [derivation_recs[0].puzzle_hash] phs_2 = [32 * bytes([1]), derivation_recs[0].puzzle_hash] phs_3 = [derivation_recs[0].puzzle_hash, 32 * bytes([1])] phs_4 = [32 * bytes([1]), 32 * bytes([2])] phs_5 = [] assert await db.one_of_puzzle_hashes_exists(phs_1) is True assert await db.one_of_puzzle_hashes_exists(phs_2) is True assert await db.one_of_puzzle_hashes_exists(phs_3) is True assert await db.one_of_puzzle_hashes_exists(phs_4) is False assert await db.one_of_puzzle_hashes_exists(phs_5) is False assert await db.index_for_pubkey(derivation_recs[4].pubkey) == 2 assert await db.index_for_puzzle_hash(derivation_recs[2].puzzle_hash) == 1 assert await db.wallet_info_for_puzzle_hash(derivation_recs[2].puzzle_hash) == ( derivation_recs[2].wallet_id, derivation_recs[2].wallet_type, ) assert len((await db.get_all_puzzle_hashes())) == 2000 assert await db.get_last_derivation_path() == 999 assert await db.get_unused_derivation_path() == 0 assert await db.get_derivation_record(0, 2, False) == derivation_recs[1] # Indeces up to 250 await db.set_used_up_to(249) assert await db.get_unused_derivation_path() == 250 except Exception as e: print(e, type(e)) await db._clear_database() await db.close() db_filename.unlink() raise e await db._clear_database() await db.close() db_filename.unlink()
async def convert_v1_to_v2(in_path: Path, out_path: Path) -> None: import aiosqlite from chia.util.db_wrapper import DBWrapper if out_path.exists(): print(f"output file already exists. {out_path}") raise RuntimeError("already exists") print(f"opening file for reading: {in_path}") async with aiosqlite.connect(in_path) as in_db: try: async with in_db.execute( "SELECT * from database_version") as cursor: row = await cursor.fetchone() if row is not None and row[0] != 1: print( f"blockchain database already version {row[0]}\nDone") raise RuntimeError("already v2") except aiosqlite.OperationalError: pass store_v1 = await BlockStore.create(DBWrapper(in_db, db_version=1)) print(f"opening file for writing: {out_path}") async with aiosqlite.connect(out_path) as out_db: await out_db.execute("pragma journal_mode=OFF") await out_db.execute("pragma synchronous=OFF") await out_db.execute("pragma cache_size=131072") await out_db.execute("pragma locking_mode=exclusive") print("initializing v2 version") await out_db.execute("CREATE TABLE database_version(version int)") await out_db.execute("INSERT INTO database_version VALUES(?)", (2, )) print("initializing v2 block store") await out_db.execute("CREATE TABLE full_blocks(" "header_hash blob PRIMARY KEY," "prev_hash blob," "height bigint," "sub_epoch_summary blob," "is_fully_compactified tinyint," "in_main_chain tinyint," "block blob," "block_record blob)") await out_db.execute("CREATE TABLE sub_epoch_segments_v3(" "ses_block_hash blob PRIMARY KEY," "challenge_segments blob)") await out_db.execute( "CREATE TABLE current_peak(key int PRIMARY KEY, hash blob)") peak_hash, peak_height = await store_v1.get_peak() print(f"peak: {peak_hash.hex()} height: {peak_height}") await out_db.execute("INSERT INTO current_peak VALUES(?, ?)", (0, peak_hash)) await out_db.commit() print("[1/5] converting full_blocks") height = peak_height + 1 hh = peak_hash commit_in = BLOCK_COMMIT_RATE rate = 1.0 start_time = time() block_start_time = start_time block_values = [] async with in_db.execute( "SELECT header_hash, prev_hash, block, sub_epoch_summary FROM block_records ORDER BY height DESC" ) as cursor: async with in_db.execute( "SELECT header_hash, height, is_fully_compactified, block FROM full_blocks ORDER BY height DESC" ) as cursor_2: await out_db.execute("begin transaction") async for row in cursor: header_hash = bytes.fromhex(row[0]) if header_hash != hh: continue # progress cursor_2 until we find the header hash while True: row_2 = await cursor_2.fetchone() if row_2 is None: print( f"ERROR: could not find block {hh.hex()}") raise RuntimeError( f"block {hh.hex()} not found") if bytes.fromhex(row_2[0]) == hh: break assert row_2[1] == height - 1 height = row_2[1] is_fully_compactified = row_2[2] block_bytes = row_2[3] prev_hash = bytes.fromhex(row[1]) block_record = row[2] ses = row[3] block_values.append(( hh, prev_hash, height, ses, is_fully_compactified, 1, # in_main_chain zstd.compress(block_bytes), block_record, )) hh = prev_hash if (height % 1000) == 0: print( f"\r{height: 10d} {(peak_height-height)*100/peak_height:.2f}% " f"{rate:0.1f} blocks/s ETA: {height//rate} s ", end="", ) sys.stdout.flush() commit_in -= 1 if commit_in == 0: commit_in = BLOCK_COMMIT_RATE await out_db.executemany( "INSERT OR REPLACE INTO full_blocks VALUES(?, ?, ?, ?, ?, ?, ?, ?)", block_values) await out_db.commit() await out_db.execute("begin transaction") block_values = [] end_time = time() rate = BLOCK_COMMIT_RATE / (end_time - start_time) start_time = end_time await out_db.executemany( "INSERT OR REPLACE INTO full_blocks VALUES(?, ?, ?, ?, ?, ?, ?, ?)", block_values) await out_db.commit() end_time = time() print( f"\r {end_time - block_start_time:.2f} seconds " ) print("[2/5] converting sub_epoch_segments_v3") commit_in = SES_COMMIT_RATE ses_values = [] ses_start_time = time() async with in_db.execute( "SELECT ses_block_hash, challenge_segments FROM sub_epoch_segments_v3" ) as cursor: count = 0 await out_db.execute("begin transaction") async for row in cursor: block_hash = bytes32.fromhex(row[0]) ses = row[1] ses_values.append((block_hash, ses)) count += 1 if (count % 100) == 0: print(f"\r{count:10d} ", end="") sys.stdout.flush() commit_in -= 1 if commit_in == 0: commit_in = SES_COMMIT_RATE await out_db.executemany( "INSERT INTO sub_epoch_segments_v3 VALUES (?, ?)", ses_values) await out_db.commit() await out_db.execute("begin transaction") ses_values = [] await out_db.executemany( "INSERT INTO sub_epoch_segments_v3 VALUES (?, ?)", ses_values) await out_db.commit() end_time = time() print( f"\r {end_time - ses_start_time:.2f} seconds " ) print("[3/5] converting hint_store") commit_in = HINT_COMMIT_RATE hint_start_time = time() hint_values = [] await out_db.execute( "CREATE TABLE hints(coin_id blob, hint blob, UNIQUE (coin_id, hint))" ) await out_db.commit() async with in_db.execute( "SELECT coin_id, hint FROM hints") as cursor: count = 0 await out_db.execute("begin transaction") async for row in cursor: hint_values.append((row[0], row[1])) commit_in -= 1 if commit_in == 0: commit_in = HINT_COMMIT_RATE await out_db.executemany( "INSERT OR IGNORE INTO hints VALUES(?, ?) ON CONFLICT DO NOTHING", hint_values) await out_db.commit() await out_db.execute("begin transaction") hint_values = [] await out_db.executemany( "INSERT OR IGNORE INTO hints VALUES (?, ?)", hint_values) await out_db.commit() end_time = time() print( f"\r {end_time - hint_start_time:.2f} seconds " ) print("[4/5] converting coin_store") await out_db.execute( "CREATE TABLE coin_record(" "coin_name blob PRIMARY KEY," " confirmed_index bigint," " spent_index bigint," # if this is zero, it means the coin has not been spent " coinbase int," " puzzle_hash blob," " coin_parent blob," " amount blob," # we use a blob of 8 bytes to store uint64 " timestamp bigint)") await out_db.commit() commit_in = COIN_COMMIT_RATE rate = 1.0 start_time = time() coin_values = [] coin_start_time = start_time async with in_db.execute( "SELECT coin_name, confirmed_index, spent_index, coinbase, puzzle_hash, coin_parent, amount, timestamp " "FROM coin_record WHERE confirmed_index <= ?", (peak_height, ), ) as cursor: count = 0 await out_db.execute("begin transaction") async for row in cursor: spent_index = row[2] # in order to convert a consistent snapshot of the # blockchain state, any coin that was spent *after* our # cutoff must be converted into an unspent coin if spent_index > peak_height: spent_index = 0 coin_values.append(( bytes.fromhex(row[0]), row[1], spent_index, row[3], bytes.fromhex(row[4]), bytes.fromhex(row[5]), row[6], row[7], )) count += 1 if (count % 2000) == 0: print( f"\r{count//1000:10d}k coins {rate:0.1f} coins/s ", end="") sys.stdout.flush() commit_in -= 1 if commit_in == 0: commit_in = COIN_COMMIT_RATE await out_db.executemany( "INSERT INTO coin_record VALUES(?, ?, ?, ?, ?, ?, ?, ?)", coin_values) await out_db.commit() await out_db.execute("begin transaction") coin_values = [] end_time = time() rate = COIN_COMMIT_RATE / (end_time - start_time) start_time = end_time await out_db.executemany( "INSERT INTO coin_record VALUES(?, ?, ?, ?, ?, ?, ?, ?)", coin_values) await out_db.commit() end_time = time() print( f"\r {end_time - coin_start_time:.2f} seconds " ) print("[5/5] build indices") index_start_time = time() print(" block store") await BlockStore.create(DBWrapper(out_db, db_version=2)) print(" coin store") await CoinStore.create(DBWrapper(out_db, db_version=2)) print(" hint store") await HintStore.create(DBWrapper(out_db, db_version=2)) end_time = time() print( f"\r {end_time - index_start_time:.2f} seconds " )
async def test_store(self): db_filename = Path("wallet_store_test.db") if db_filename.exists(): db_filename.unlink() db_connection = await aiosqlite.connect(db_filename) db_wrapper = DBWrapper(db_connection) store = await WalletPoolStore.create(db_wrapper) try: await db_wrapper.begin_transaction() coin_0 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) coin_0_alt = Coin(token_bytes(32), token_bytes(32), uint64(12312)) solution_0: CoinSpend = make_child_solution(None, coin_0) solution_0_alt: CoinSpend = make_child_solution(None, coin_0_alt) solution_1: CoinSpend = make_child_solution(solution_0) assert store.get_spends_for_wallet(0) == [] assert store.get_spends_for_wallet(1) == [] await store.add_spend(1, solution_1, 100) assert store.get_spends_for_wallet(1) == [(100, solution_1)] # Idempotent await store.add_spend(1, solution_1, 100) assert store.get_spends_for_wallet(1) == [(100, solution_1)] with pytest.raises(ValueError): await store.add_spend(1, solution_1, 101) # Rebuild cache, no longer present await db_wrapper.rollback_transaction() await store.rebuild_cache() assert store.get_spends_for_wallet(1) == [] await store.rebuild_cache() await store.add_spend(1, solution_1, 100) assert store.get_spends_for_wallet(1) == [(100, solution_1)] solution_1_alt: CoinSpend = make_child_solution(solution_0_alt) with pytest.raises(ValueError): await store.add_spend(1, solution_1_alt, 100) assert store.get_spends_for_wallet(1) == [(100, solution_1)] solution_2: CoinSpend = make_child_solution(solution_1) await store.add_spend(1, solution_2, 100) await store.rebuild_cache() solution_3: CoinSpend = make_child_solution(solution_2) await store.add_spend(1, solution_3, 100) solution_4: CoinSpend = make_child_solution(solution_3) with pytest.raises(ValueError): await store.add_spend(1, solution_4, 99) await store.rebuild_cache() await store.add_spend(1, solution_4, 101) await store.rebuild_cache() await store.rollback(101, 1) await store.rebuild_cache() assert store.get_spends_for_wallet(1) == [ (100, solution_1), (100, solution_2), (100, solution_3), (101, solution_4), ] await store.rebuild_cache() await store.rollback(100, 1) await store.rebuild_cache() assert store.get_spends_for_wallet(1) == [ (100, solution_1), (100, solution_2), (100, solution_3), ] with pytest.raises(ValueError): await store.add_spend(1, solution_1, 105) await store.add_spend(1, solution_4, 105) solution_5: CoinSpend = make_child_solution(solution_4) await store.add_spend(1, solution_5, 105) await store.rollback(99, 1) assert store.get_spends_for_wallet(1) == [] finally: await db_connection.close() db_filename.unlink()