def _compact_hashX(self, hashX, hist_map, hist_list, write_items, keys_to_delete): '''Compres history for a hashX. hist_list is an ordered list of the histories to be compressed.''' # History entries (tx numbers) are 4 bytes each. Distribute # over rows of up to 50KB in size. A fixed row size means # future compactions will not need to update the first N - 1 # rows. max_row_size = self.max_hist_row_entries * 4 full_hist = b''.join(hist_list) nrows = (len(full_hist) + max_row_size - 1) // max_row_size if nrows > 4: self.logger.info('hashX {} is large: {:,d} entries across ' '{:,d} rows'.format(hash_to_str(hashX), len(full_hist) // 4, nrows)) # Find what history needs to be written, and what keys need to # be deleted. Start by assuming all keys are to be deleted, # and then remove those that are the same on-disk as when # compacted. write_size = 0 keys_to_delete.update(hist_map) for n, chunk in enumerate(util.chunks(full_hist, max_row_size)): key = hashX + pack('>H', n) if hist_map.get(key) == chunk: keys_to_delete.remove(key) else: write_items.append((key, chunk)) write_size += len(chunk) assert n + 1 == nrows self.comp_flush_count = max(self.comp_flush_count, n) return write_size
async def reorg_chain(self, count=None): '''Handle a chain reorganisation. Count is the number of blocks to simulate a reorg, or None for a real reorg.''' if count is None: self.logger.info('chain reorg detected') else: self.logger.info(f'faking a reorg of {count:,d} blocks') await self.flush(True) async def get_raw_blocks(last_height, hex_hashes): heights = range(last_height, last_height - len(hex_hashes), -1) try: blocks = [self.db.read_raw_block(height) for height in heights] self.logger.info(f'read {len(blocks)} blocks from disk') return blocks except FileNotFoundError: return await self.daemon.raw_blocks(hex_hashes) def flush_backup(): # self.touched can include other addresses which is # harmless, but remove None. self.touched.discard(None) self.db.flush_backup(self.flush_data(), self.touched) _start, last, hashes = await self.reorg_hashes(count) # Reverse and convert to hex strings. hashes = [hash_to_hex_str(hash) for hash in reversed(hashes)] for hex_hashes in chunks(hashes, 50): raw_blocks = await get_raw_blocks(last, hex_hashes) await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks) await self.run_in_thread_with_lock(flush_backup) last -= len(raw_blocks) await self.prefetcher.reset_height(self.height)
async def reorg_chain(self, count=None): '''Handle a chain reorganisation. Count is the number of blocks to simulate a reorg, or None for a real reorg.''' if count is None: self.logger.info('chain reorg detected') else: self.logger.info(f'faking a reorg of {count:,d} blocks') await self.tasks.run_in_thread(self.flush, True) async def get_raw_blocks(last_height, hex_hashes): heights = range(last_height, last_height - len(hex_hashes), -1) try: blocks = [self.read_raw_block(height) for height in heights] self.logger.info(f'read {len(blocks)} blocks from disk') return blocks except Exception: return await self.daemon.raw_blocks(hex_hashes) start, hashes = await self.reorg_hashes(count) # Reverse and convert to hex strings. hashes = [hash_to_hex_str(hash) for hash in reversed(hashes)] last = start + count - 1 for hex_hashes in chunks(hashes, 50): raw_blocks = await get_raw_blocks(last, hex_hashes) async with self.state_lock: await self.tasks.run_in_thread(self.backup_blocks, raw_blocks) last -= len(raw_blocks) # Truncate header_mc: header count is 1 more than the height self.header_mc.truncate(self.height + 1) await self.prefetcher.reset_height(self.height)
def flush_backup(self, flush_data: FlushData, touched_hashxs): '''Like flush_dbs() but when backing up. All UTXOs are flushed.''' assert not flush_data.headers assert not flush_data.block_tx_hashes assert flush_data.height < self.db_height self.history.assert_flushed() assert len(flush_data.undo_block_tx_hashes ) == self.db_height - flush_data.height start_time = time.time() tx_delta = flush_data.tx_count - self.last_flush_tx_count tx_hashes = [] for block in flush_data.undo_block_tx_hashes: tx_hashes += [*util.chunks(block, 32)] flush_data.undo_block_tx_hashes.clear() assert len(tx_hashes) == -tx_delta self.backup_fs(flush_data.height, flush_data.tx_count) self.history.backup( hashXs=touched_hashxs, tx_count=flush_data.tx_count, tx_hashes=tx_hashes, spends=flush_data.undo_historical_spends, ) flush_data.undo_historical_spends.clear() with self.utxo_db.write_batch() as batch: self.flush_utxo_db(batch, flush_data) # Flush state last as it reads the wall time. self.flush_state(batch) elapsed = self.last_flush - start_time self.logger.info(f'backup flush took ' f'{elapsed:.1f}s. Height {flush_data.height:,d} ' f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})')
async def _process_mempool(self, all_hashes, touched, mempool_height): # Re-sync with the new set of hashes txs = self.txs hashXs = self.hashXs if mempool_height != self.api.db_height(): raise DBSyncError # First handle txs that have disappeared for tx_hash in (set(txs) - all_hashes): tx = txs.pop(tx_hash) tx_hashXs = {hashX for hashX, value in tx.in_pairs} tx_hashXs.update(hashX for hashX, value in tx.out_pairs) for hashX in tx_hashXs: hashXs[hashX].remove(tx_hash) if not hashXs[hashX]: del hashXs[hashX] touched |= tx_hashXs # Process new transactions new_hashes = list(all_hashes.difference(txs)) if new_hashes: group = TaskGroup() for hashes in chunks(new_hashes, 200): coro = self._fetch_and_accept(hashes, all_hashes, touched) await group.spawn(coro) if mempool_height != self.api.db_height(): raise DBSyncError tx_map = {} utxo_map = {} async for task in group: deferred, unspent = task.result() tx_map.update(deferred) utxo_map.update(unspent) prior_count = 0 # FIXME: this is not particularly efficient while tx_map and len(tx_map) != prior_count: prior_count = len(tx_map) tx_map, utxo_map = self._accept_transactions( tx_map, utxo_map, touched) if tx_map: self.logger.error(f'{len(tx_map)} txs dropped') return touched
async def _process_mempool(self, all_hashes): # Re-sync with the new set of hashes txs = self.txs hashXs = self.hashXs touched = set() # First handle txs that have disappeared for tx_hash in set(txs).difference(all_hashes): tx = txs.pop(tx_hash) tx_hashXs = set(hashX for hashX, value in tx.in_pairs) tx_hashXs.update(hashX for hashX, value in tx.out_pairs) for hashX in tx_hashXs: hashXs[hashX].remove(tx_hash) if not hashXs[hashX]: del hashXs[hashX] touched.update(tx_hashXs) # Process new transactions new_hashes = list(all_hashes.difference(txs)) jobs = [ self.tasks.create_task( self._fetch_and_accept(hashes, all_hashes, touched)) for hashes in chunks(new_hashes, 2000) ] if jobs: await asyncio.wait(jobs) tx_map = {} utxo_map = {} for job in jobs: deferred, unspent = job.result() tx_map.update(deferred) utxo_map.update(unspent) # Handle the stragglers if len(tx_map) >= 10: self.logger.info(f'{len(tx_map)} stragglers') prior_count = 0 # FIXME: this is not particularly efficient while tx_map and len(tx_map) != prior_count: prior_count = len(tx_map) tx_map, utxo_map = self._accept_transactions( tx_map, utxo_map, touched) if tx_map: self.logger.info(f'{len(tx_map)} txs dropped') return touched
async def reorg_chain(self, count=None): '''Handle a chain reorganisation. Count is the number of blocks to simulate a reorg, or None for a real reorg.''' if count is None: self.logger.info('chain reorg detected') else: self.logger.info('faking a reorg of {:,d} blocks'.format(count)) await self.controller.run_in_executor(self.flush, True) hashes = await self.reorg_hashes(count) # Reverse and convert to hex strings. hashes = [hash_to_str(hash) for hash in reversed(hashes)] for hex_hashes in chunks(hashes, 50): blocks = await self.daemon.raw_blocks(hex_hashes) await self.controller.run_in_executor(self.backup_blocks, blocks) await self.prefetcher.reset_height()
async def reorg_chain(self, count=None): '''Handle a chain reorganisation. Count is the number of blocks to simulate a reorg, or None for a real reorg.''' if count is None: self.logger.info('chain reorg detected') else: self.logger.info(f'faking a reorg of {count:,d} blocks') await run_in_thread(self.flush, True) async def get_raw_blocks(last_height, hex_hashes): heights = range(last_height, last_height - len(hex_hashes), -1) try: blocks = [self.read_raw_block(height) for height in heights] self.logger.info(f'read {len(blocks)} blocks from disk') return blocks except Exception: return await self.daemon.raw_blocks(hex_hashes) start, last, hashes = await self.reorg_hashes(count) # Reverse and convert to hex strings. hashes = [hash_to_hex_str(hash) for hash in reversed(hashes)] # get saved evntlog hashYs if hashes: eventlog_hashYs = reduce( operator.add, [self.get_block_hashYs(x) for x in hashes]) else: eventlog_hashYs = [] self.logger.info('chain reorg eventlog_hashYs {} {}'.format( eventlog_hashYs, hashes)) for hex_hashes in chunks(hashes, 50): raw_blocks = await get_raw_blocks(last, hex_hashes) await self.run_in_thread_shielded(self.backup_blocks_eventlogs, raw_blocks, eventlog_hashYs) last -= len(raw_blocks) # Truncate header_mc: header count is 1 more than the height. # Note header_mc is None if the reorg happens at startup. if self.header_mc: self.header_mc.truncate(self.height + 1) await self.prefetcher.reset_height(self.height)
def test_chunks(): assert list(util.chunks([1, 2, 3, 4, 5], 2)) == [[1, 2], [3, 4], [5]]
async def _process_mempool(self, all_hashes, touched, mempool_height): # Re-sync with the new set of hashes txs = self.txs hashXs = self.hashXs tx_to_create = self.tx_to_asset_create tx_to_reissue = self.tx_to_asset_reissue creates = self.asset_creates reissues = self.asset_reissues if mempool_height != self.api.db_height(): raise DBSyncError # First handle txs that have disappeared for tx_hash in set(txs).difference(all_hashes): tx = txs.pop(tx_hash) reissued_asset = tx_to_reissue.pop(tx_hash, None) if reissued_asset: del reissues[reissued_asset] created_asset = tx_to_create.pop(tx_hash, None) if created_asset: del creates[created_asset] tx_hashXs = set(hashX for hashX, value, _, _ in tx.in_pairs) tx_hashXs.update(hashX for hashX, value, _, _ in tx.out_pairs) for hashX in tx_hashXs: hashXs[hashX].remove(tx_hash) if not hashXs[hashX]: del hashXs[hashX] touched.update(tx_hashXs) # Process new transactions new_hashes = list(all_hashes.difference(txs)) if new_hashes: group = TaskGroup() for hashes in chunks(new_hashes, 200): coro = self._fetch_and_accept(hashes, all_hashes, touched) await group.spawn(coro) tx_map = {} utxo_map = {} async for task in group: (deferred, unspent), internal_creates, internal_reissues = task.result() # Store asset changes for asset, stats in internal_creates.items(): tx_to_create[hex_str_to_hash( stats['source']['tx_hash'])] = asset creates[asset] = stats for asset, stats in internal_reissues.items(): tx_to_reissue[hex_str_to_hash( stats['source']['tx_hash'])] = asset reissues[asset] = stats tx_map.update(deferred) utxo_map.update(unspent) prior_count = 0 # FIXME: this is not particularly efficient while tx_map and len(tx_map) != prior_count: prior_count = len(tx_map) tx_map, utxo_map = self._accept_transactions( tx_map, utxo_map, touched) if tx_map: self.logger.error(f'{len(tx_map)} txs dropped') return touched
async def _process_mempool( self, *, all_hashes: Set[bytes], # set of txids touched_hashxs: Set[bytes], # set of hashXs touched_outpoints: Set[Tuple[bytes, int]], # set of outpoints mempool_height: int, ) -> None: # Re-sync with the new set of hashes txs = self.txs hashXs = self.hashXs txo_to_spender = self.txo_to_spender if mempool_height != self.api.db_height(): raise DBSyncError # First handle txs that have disappeared for tx_hash in (set(txs) - all_hashes): tx = txs.pop(tx_hash) # hashXs tx_hashXs = {hashX for hashX, value in tx.in_pairs} tx_hashXs.update(hashX for hashX, value in tx.out_pairs) for hashX in tx_hashXs: hashXs[hashX].remove(tx_hash) if not hashXs[hashX]: del hashXs[hashX] touched_hashxs |= tx_hashXs # outpoints for prevout in tx.prevouts: del txo_to_spender[prevout] touched_outpoints.add(prevout) for out_idx, out_pair in enumerate(tx.out_pairs): touched_outpoints.add((tx_hash, out_idx)) # Process new transactions new_hashes = list(all_hashes.difference(txs)) if new_hashes: group = TaskGroup() for hashes in chunks(new_hashes, 200): coro = self._fetch_and_accept( hashes=hashes, all_hashes=all_hashes, touched_hashxs=touched_hashxs, touched_outpoints=touched_outpoints, ) await group.spawn(coro) if mempool_height != self.api.db_height(): raise DBSyncError tx_map = {} utxo_map = {} async for task in group: deferred, unspent = task.result() tx_map.update(deferred) utxo_map.update(unspent) prior_count = 0 # FIXME: this is not particularly efficient while tx_map and len(tx_map) != prior_count: prior_count = len(tx_map) tx_map, utxo_map = self._accept_transactions( tx_map=tx_map, utxo_map=utxo_map, touched_hashxs=touched_hashxs, touched_outpoints=touched_outpoints, ) if tx_map: self.logger.error(f'{len(tx_map)} txs dropped')