async def time_it():
source = Source(500000).hashes
cp_length = 492000
import time
cache = MerkleCache(merkle, source)
await cache.initialize(cp_length)
cp_hashes = await source(0, cp_length)
brs2 = []
t1 = time.monotonic()
for index in range(5, 400000, 500):
brs2.append(await cache.branch_and_root(cp_length, index))
t2 = time.monotonic()
print(t2 - t1)
assert False
async def test_merkle_cache_bad():
length = 23
source = Source(length).hashes
cache = MerkleCache(merkle, source)
await cache.initialize(length)
await cache.branch_and_root(5, 3)
with pytest.raises(TypeError):
await cache.branch_and_root(5.0, 3)
with pytest.raises(TypeError):
await cache.branch_and_root(5, 3.0)
with pytest.raises(ValueError):
await cache.branch_and_root(0, -1)
with pytest.raises(ValueError):
await cache.branch_and_root(3, 3)
async def test_merkle_cache_extension():
source = Source(64).hashes
for length in range(14, 18):
for cp_length in range(30, 36):
cache = MerkleCache(merkle, source)
await cache.initialize(length)
cp_hashes = await source(0, cp_length)
# All possible indices
for index in range(cp_length):
# Compare correct answer with cache
branch, root = merkle.branch_and_root(cp_hashes, index)
branch2, root2 = await cache.branch_and_root(cp_length, index)
assert branch == branch2
assert root == root2
def __init__(self, env):
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.env = env
self.coin = env.coin
# Setup block header size handlers
if self.coin.STATIC_BLOCK_HEADERS:
self.header_offset = self.coin.static_header_offset
self.header_len = self.coin.static_header_len
else:
self.header_offset = self.dynamic_header_offset
self.header_len = self.dynamic_header_len
self.logger.info(f'switching current directory to {env.db_dir}')
os.chdir(env.db_dir)
self.db_class = db_class(self.env.db_engine)
self.history = History()
self.eventlog = Eventlog()
self.unflushed_hashYs = defaultdict(set) # {blockHash => [hashY, ]}, for reorg_chain
self.hashY_db = None
self.utxo_db = None
self.tx_counts = None
self.last_flush = time.time()
self.logger.info(f'using {self.env.db_engine} for DB backend')
# Header merkle cache
self.merkle = Merkle()
self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes)
self.headers_file = util.LogicalFile('meta/headers', 2, 16000000)
self.tx_counts_file = util.LogicalFile('meta/txcounts', 2, 2000000)
self.hashes_file = util.LogicalFile('meta/hashes', 4, 16000000)
if not self.coin.STATIC_BLOCK_HEADERS:
self.headers_offsets_file = util.LogicalFile(
'meta/headers_offsets', 2, 16000000)
async def test_merkle_cache():
lengths = (*range(1, 18), 31, 32, 33, 57)
source = Source(max(lengths)).hashes
for length in lengths:
cache = MerkleCache(merkle, source)
await cache.initialize(length)
# Simulate all possible checkpoints
for cp_length in range(1, length + 1):
cp_hashes = await source(0, cp_length)
# All possible indices
for index in range(cp_length):
# Compare correct answer with cache
branch, root = merkle.branch_and_root(cp_hashes, index)
branch2, root2 = await cache.branch_and_root(cp_length, index)
assert branch == branch2
assert root == root2
async def _first_caught_up(self):
self.logger.info(f'caught up to height {self.height}')
# Flush everything but with first_sync->False state.
first_sync = self.first_sync
self.first_sync = False
self.flush(True)
if first_sync:
self.logger.info(f'{electrumx.version} synced to '
f'height {self.height:,d}')
# Initialise the notification framework
await self.notifications.on_block(set(), set(), self.height)
# Reopen for serving
await self.open_for_serving()
# Populate the header merkle cache
length = max(1, self.height - self.env.reorg_limit)
self.header_mc = MerkleCache(self.merkle, HeaderSource(self), length)
self.logger.info('populated header merkle cache')
class DB:
'''Simple wrapper of the backend database for querying.
Performs no DB update, though the DB will be cleaned on opening if
it was shutdown uncleanly.
'''
DB_VERSIONS = (6, 7, 8)
utxo_db: Optional['Storage']
class DBError(Exception):
'''Raised on general DB errors generally indicating corruption.'''
def __init__(self, env: 'Env'):
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.env = env
self.coin = env.coin
# Setup block header size handlers
if self.coin.STATIC_BLOCK_HEADERS:
self.header_offset = self.coin.static_header_offset
self.header_len = self.coin.static_header_len
else:
self.header_offset = self.dynamic_header_offset
self.header_len = self.dynamic_header_len
self.logger.info(f'switching current directory to {env.db_dir}')
os.chdir(env.db_dir)
self.db_class = db_class(self.env.db_engine)
self.history = History()
# Key: b'u' + address_hashX + txout_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer (in satoshis)
# "at address, at outpoint, there is a UTXO of value v"
# ---
# Key: b'h' + compressed_tx_hash + txout_idx + tx_num
# Value: hashX
# "some outpoint created a UTXO at address"
# ---
# Key: b'U' + block_height
# Value: byte-concat list of (hashX + tx_num + value_sats)
# "undo data: list of UTXOs spent at block height"
self.utxo_db = None
self.utxo_flush_count = 0
self.fs_height = -1
self.fs_tx_count = 0
self.db_height = -1
self.db_tx_count = 0
self.db_tip = None # type: Optional[bytes]
self.tx_counts = None
self.last_flush = time.time()
self.last_flush_tx_count = 0
self.wall_time = 0
self.first_sync = True
self.db_version = -1
self.logger.info(f'using {self.env.db_engine} for DB backend')
# Header merkle cache
self.merkle = Merkle()
self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes)
# on-disk: raw block headers in chain order
self.headers_file = util.LogicalFile('meta/headers', 2, 16000000)
# on-disk: cumulative number of txs at the end of height N
self.tx_counts_file = util.LogicalFile('meta/txcounts', 2, 2000000)
# on-disk: 32 byte txids in chain order, allows (tx_num -> txid) map
self.hashes_file = util.LogicalFile('meta/hashes', 4, 16000000)
if not self.coin.STATIC_BLOCK_HEADERS:
self.headers_offsets_file = util.LogicalFile(
'meta/headers_offsets', 2, 16000000)
async def _read_tx_counts(self):
if self.tx_counts is not None:
return
# tx_counts[N] has the cumulative number of txs at the end of
# height N. So tx_counts[0] is 1 - the genesis coinbase
size = (self.db_height + 1) * 8
tx_counts = self.tx_counts_file.read(0, size)
assert len(tx_counts) == size
self.tx_counts = array('Q', tx_counts)
if self.tx_counts:
assert self.db_tx_count == self.tx_counts[-1]
else:
assert self.db_tx_count == 0
async def _open_dbs(self, for_sync: bool, compacting: bool):
assert self.utxo_db is None
# First UTXO DB
self.utxo_db = self.db_class('utxo', for_sync)
if self.utxo_db.is_new:
self.logger.info('created new database')
self.logger.info('creating metadata directory')
os.mkdir('meta')
with util.open_file('COIN', create=True) as f:
f.write(f'ElectrumX databases and metadata for '
f'{self.coin.NAME} {self.coin.NET}'.encode())
if not self.coin.STATIC_BLOCK_HEADERS:
self.headers_offsets_file.write(0, b'\0\0\0\0\0\0\0\0')
else:
self.logger.info(f'opened UTXO DB (for sync: {for_sync})')
self.read_utxo_state()
# Then history DB
self.utxo_flush_count = self.history.open_db(self.db_class, for_sync,
self.utxo_flush_count,
compacting)
self.clear_excess_undo_info()
# Read TX counts (requires meta directory)
await self._read_tx_counts()
async def open_for_compacting(self):
await self._open_dbs(True, True)
async def open_for_sync(self):
'''Open the databases to sync to the daemon.
When syncing we want to reserve a lot of open files for the
synchronization. When serving clients we want the open files for
serving network connections.
'''
await self._open_dbs(True, False)
async def open_for_serving(self):
'''Open the databases for serving. If they are already open they are
closed first.
'''
if self.utxo_db:
self.logger.info('closing DBs to re-open for serving')
self.utxo_db.close()
self.history.close_db()
self.utxo_db = None
await self._open_dbs(False, False)
# Header merkle cache
async def populate_header_merkle_cache(self):
self.logger.info('populating header merkle cache...')
length = max(1, self.db_height - self.env.reorg_limit)
start = time.time()
await self.header_mc.initialize(length)
elapsed = time.time() - start
self.logger.info(f'header merkle cache populated in {elapsed:.1f}s')
async def header_branch_and_root(self, length, height):
return await self.header_mc.branch_and_root(length, height)
# Flushing
def assert_flushed(self, flush_data):
'''Asserts state is fully flushed.'''
assert flush_data.tx_count == self.fs_tx_count == self.db_tx_count
assert flush_data.height == self.fs_height == self.db_height
assert flush_data.tip == self.db_tip
assert not flush_data.headers
assert not flush_data.block_tx_hashes
assert not flush_data.adds
assert not flush_data.deletes
assert not flush_data.undo_infos
self.history.assert_flushed()
def flush_dbs(self, flush_data, flush_utxos, estimate_txs_remaining):
'''Flush out cached state. History is always flushed; UTXOs are
flushed if flush_utxos.'''
if flush_data.height == self.db_height:
self.assert_flushed(flush_data)
return
start_time = time.time()
prior_flush = self.last_flush
tx_delta = flush_data.tx_count - self.last_flush_tx_count
# Flush to file system
self.flush_fs(flush_data)
# Then history
self.flush_history()
# Flush state last as it reads the wall time.
with self.utxo_db.write_batch() as batch:
if flush_utxos:
self.flush_utxo_db(batch, flush_data)
self.flush_state(batch)
# Update and put the wall time again - otherwise we drop the
# time it took to commit the batch
self.flush_state(self.utxo_db)
elapsed = self.last_flush - start_time
self.logger.info(f'flush #{self.history.flush_count:,d} took '
f'{elapsed:.1f}s. Height {flush_data.height:,d} '
f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})')
# Catch-up stats
if self.utxo_db.for_sync:
flush_interval = self.last_flush - prior_flush
tx_per_sec_gen = int(flush_data.tx_count / self.wall_time)
tx_per_sec_last = 1 + int(tx_delta / flush_interval)
eta = estimate_txs_remaining() / tx_per_sec_last
self.logger.info(f'tx/sec since genesis: {tx_per_sec_gen:,d}, '
f'since last flush: {tx_per_sec_last:,d}')
self.logger.info(f'sync time: {formatted_time(self.wall_time)} '
f'ETA: {formatted_time(eta)}')
def flush_fs(self, flush_data):
'''Write headers, tx counts and block tx hashes to the filesystem.
The first height to write is self.fs_height + 1. The FS
metadata is all append-only, so in a crash we just pick up
again from the height stored in the DB.
'''
prior_tx_count = (self.tx_counts[self.fs_height]
if self.fs_height >= 0 else 0)
assert len(flush_data.block_tx_hashes) == len(flush_data.headers)
assert flush_data.height == self.fs_height + len(flush_data.headers)
assert flush_data.tx_count == (self.tx_counts[-1]
if self.tx_counts else 0)
assert len(self.tx_counts) == flush_data.height + 1
hashes = b''.join(flush_data.block_tx_hashes)
flush_data.block_tx_hashes.clear()
assert len(hashes) % 32 == 0
assert len(hashes) // 32 == flush_data.tx_count - prior_tx_count
# Write the headers, tx counts, and tx hashes
start_time = time.time()
height_start = self.fs_height + 1
offset = self.header_offset(height_start)
self.headers_file.write(offset, b''.join(flush_data.headers))
self.fs_update_header_offsets(offset, height_start, flush_data.headers)
flush_data.headers.clear()
offset = height_start * self.tx_counts.itemsize
self.tx_counts_file.write(offset,
self.tx_counts[height_start:].tobytes())
offset = prior_tx_count * 32
self.hashes_file.write(offset, hashes)
self.fs_height = flush_data.height
self.fs_tx_count = flush_data.tx_count
if self.utxo_db.for_sync:
elapsed = time.time() - start_time
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
def flush_history(self):
self.history.flush()
def flush_utxo_db(self, batch, flush_data: FlushData):
'''Flush the cached DB writes and UTXO set to the batch.'''
# Care is needed because the writes generated by flushing the
# UTXO state may have keys in common with our write cache or
# may be in the DB already.
start_time = time.time()
add_count = len(flush_data.adds)
spend_count = len(flush_data.deletes) // 2
# Spends
batch_delete = batch.delete
for key in sorted(flush_data.deletes):
batch_delete(key)
flush_data.deletes.clear()
# New UTXOs
batch_put = batch.put
for key, value in flush_data.adds.items():
# key: txid+out_idx, value: hashX+tx_num+value_sats
hashX = value[:HASHX_LEN]
txout_idx = key[-4:]
tx_num = value[HASHX_LEN:HASHX_LEN + TXNUM_LEN]
value_sats = value[-8:]
suffix = txout_idx + tx_num
batch_put(b'h' + key[:COMP_TXID_LEN] + suffix, hashX)
batch_put(b'u' + hashX + suffix, value_sats)
flush_data.adds.clear()
# New undo information
self.flush_undo_infos(batch_put, flush_data.undo_infos)
flush_data.undo_infos.clear()
if self.utxo_db.for_sync:
block_count = flush_data.height - self.db_height
tx_count = flush_data.tx_count - self.db_tx_count
elapsed = time.time() - start_time
self.logger.info(f'flushed {block_count:,d} blocks with '
f'{tx_count:,d} txs, {add_count:,d} UTXO adds, '
f'{spend_count:,d} spends in '
f'{elapsed:.1f}s, committing...')
self.utxo_flush_count = self.history.flush_count
self.db_height = flush_data.height
self.db_tx_count = flush_data.tx_count
self.db_tip = flush_data.tip
def flush_state(self, batch):
'''Flush chain state to the batch.'''
now = time.time()
self.wall_time += now - self.last_flush
self.last_flush = now
self.last_flush_tx_count = self.fs_tx_count
self.write_utxo_state(batch)
def flush_backup(self, flush_data, touched):
'''Like flush_dbs() but when backing up. All UTXOs are flushed.'''
assert not flush_data.headers
assert not flush_data.block_tx_hashes
assert flush_data.height < self.db_height
self.history.assert_flushed()
start_time = time.time()
tx_delta = flush_data.tx_count - self.last_flush_tx_count
self.backup_fs(flush_data.height, flush_data.tx_count)
self.history.backup(touched, flush_data.tx_count)
with self.utxo_db.write_batch() as batch:
self.flush_utxo_db(batch, flush_data)
# Flush state last as it reads the wall time.
self.flush_state(batch)
elapsed = self.last_flush - start_time
self.logger.info(f'backup flush #{self.history.flush_count:,d} took '
f'{elapsed:.1f}s. Height {flush_data.height:,d} '
f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})')
def fs_update_header_offsets(self, offset_start, height_start, headers):
if self.coin.STATIC_BLOCK_HEADERS:
return
offset = offset_start
offsets = []
for h in headers:
offset += len(h)
offsets.append(pack_le_uint64(offset))
# For each header we get the offset of the next header, hence we
# start writing from the next height
pos = (height_start + 1) * 8
self.headers_offsets_file.write(pos, b''.join(offsets))
def dynamic_header_offset(self, height):
assert not self.coin.STATIC_BLOCK_HEADERS
offset, = unpack_le_uint64(
self.headers_offsets_file.read(height * 8, 8))
return offset
def dynamic_header_len(self, height):
return self.dynamic_header_offset(height + 1)\
- self.dynamic_header_offset(height)
def backup_fs(self, height, tx_count):
'''Back up during a reorg. This just updates our pointers.'''
self.fs_height = height
self.fs_tx_count = tx_count
# Truncate header_mc: header count is 1 more than the height.
self.header_mc.truncate(height + 1)
async def raw_header(self, height):
'''Return the binary header at the given height.'''
header, n = await self.read_headers(height, 1)
if n != 1:
raise IndexError(f'height {height:,d} out of range')
return header
async def read_headers(self, start_height, count):
'''Requires start_height >= 0, count >= 0. Reads as many headers as
are available starting at start_height up to count. This
would be zero if start_height is beyond self.db_height, for
example.
Returns a (binary, n) pair where binary is the concatenated
binary headers, and n is the count of headers returned.
'''
if start_height < 0 or count < 0:
raise self.DBError(f'{count:,d} headers starting at '
f'{start_height:,d} not on disk')
def read_headers():
# Read some from disk
disk_count = max(0, min(count, self.db_height + 1 - start_height))
if disk_count:
offset = self.header_offset(start_height)
size = self.header_offset(start_height + disk_count) - offset
return self.headers_file.read(offset, size), disk_count
return b'', 0
return await run_in_thread(read_headers)
def fs_tx_hash(self, tx_num):
'''Return a pair (tx_hash, tx_height) for the given tx number.
If the tx_height is not on disk, returns (None, tx_height).'''
tx_height = bisect_right(self.tx_counts, tx_num)
if tx_height > self.db_height:
tx_hash = None
else:
tx_hash = self.hashes_file.read(tx_num * 32, 32)
return tx_hash, tx_height
def fs_tx_hashes_at_blockheight(self, block_height):
'''Return a list of tx_hashes at given block height,
in the same order as in the block.
'''
if block_height > self.db_height:
raise self.DBError(
f'block {block_height:,d} not on disk (>{self.db_height:,d})')
assert block_height >= 0
if block_height > 0:
first_tx_num = self.tx_counts[block_height - 1]
else:
first_tx_num = 0
num_txs_in_block = self.tx_counts[block_height] - first_tx_num
tx_hashes = self.hashes_file.read(first_tx_num * 32,
num_txs_in_block * 32)
assert num_txs_in_block == len(tx_hashes) // 32
return [
tx_hashes[idx * 32:(idx + 1) * 32]
for idx in range(num_txs_in_block)
]
async def tx_hashes_at_blockheight(self, block_height):
return await run_in_thread(self.fs_tx_hashes_at_blockheight,
block_height)
async def fs_block_hashes(self, height, count):
headers_concat, headers_count = await self.read_headers(height, count)
if headers_count != count:
raise self.DBError(f'only got {headers_count:,d} headers starting '
f'at {height:,d}, not {count:,d}')
offset = 0
headers = []
for n in range(count):
hlen = self.header_len(height + n)
headers.append(headers_concat[offset:offset + hlen])
offset += hlen
return [self.coin.header_hash(header) for header in headers]
async def limited_history(self, hashX, *, limit=1000):
'''Return an unpruned, sorted list of (tx_hash, height) tuples of
confirmed transactions that touched the address, earliest in
the blockchain first. Includes both spending and receiving
transactions. By default returns at most 1000 entries. Set
limit to None to get them all.
'''
def read_history():
tx_nums = list(self.history.get_txnums(hashX, limit))
fs_tx_hash = self.fs_tx_hash
return [fs_tx_hash(tx_num) for tx_num in tx_nums]
while True:
history = await run_in_thread(read_history)
if all(hash is not None for hash, height in history):
return history
self.logger.warning(f'limited_history: tx hash '
f'not found (reorg?), retrying...')
await sleep(0.25)
# -- Undo information
def min_undo_height(self, max_height):
'''Returns a height from which we should store undo info.'''
return max_height - self.env.reorg_limit + 1
def undo_key(self, height: int) -> bytes:
'''DB key for undo information at the given height.'''
return b'U' + pack_be_uint32(height)
def read_undo_info(self, height):
'''Read undo information from a file for the current height.'''
return self.utxo_db.get(self.undo_key(height))
def flush_undo_infos(self, batch_put,
undo_infos: Sequence[Tuple[Sequence[bytes], int]]):
'''undo_infos is a list of (undo_info, height) pairs.'''
for undo_info, height in undo_infos:
batch_put(self.undo_key(height), b''.join(undo_info))
def raw_block_prefix(self):
return 'meta/block'
def raw_block_path(self, height):
return f'{self.raw_block_prefix()}{height:d}'
def read_raw_block(self, height):
'''Returns a raw block read from disk. Raises FileNotFoundError
if the block isn't on-disk.'''
with util.open_file(self.raw_block_path(height)) as f:
return f.read(-1)
def write_raw_block(self, block, height):
'''Write a raw block to disk.'''
with util.open_truncate(self.raw_block_path(height)) as f:
f.write(block)
# Delete old blocks to prevent them accumulating
try:
del_height = self.min_undo_height(height) - 1
os.remove(self.raw_block_path(del_height))
except FileNotFoundError:
pass
def clear_excess_undo_info(self):
'''Clear excess undo info. Only most recent N are kept.'''
prefix = b'U'
min_height = self.min_undo_height(self.db_height)
keys = []
for key, _hist in self.utxo_db.iterator(prefix=prefix):
height, = unpack_be_uint32(key[-4:])
if height >= min_height:
break
keys.append(key)
if keys:
with self.utxo_db.write_batch() as batch:
for key in keys:
batch.delete(key)
self.logger.info(f'deleted {len(keys):,d} stale undo entries')
# delete old block files
prefix = self.raw_block_prefix()
paths = [
path for path in glob(f'{prefix}[0-9]*')
if len(path) > len(prefix) and int(path[len(prefix):]) < min_height
]
if paths:
for path in paths:
try:
os.remove(path)
except FileNotFoundError:
pass
self.logger.info(f'deleted {len(paths):,d} stale block files')
# -- UTXO database
def read_utxo_state(self):
state = self.utxo_db.get(b'state')
if not state:
self.db_height = -1
self.db_tx_count = 0
self.db_tip = b'\0' * 32
self.db_version = max(self.DB_VERSIONS)
self.utxo_flush_count = 0
self.wall_time = 0
self.first_sync = True
else:
state = ast.literal_eval(state.decode())
if not isinstance(state, dict):
raise self.DBError('failed reading state from DB')
self.db_version = state['db_version']
if self.db_version not in self.DB_VERSIONS:
raise self.DBError(
f'your UTXO DB version is {self.db_version} '
f'but this software only handles versions '
f'{self.DB_VERSIONS}')
# backwards compat
genesis_hash = state['genesis']
if isinstance(genesis_hash, bytes):
genesis_hash = genesis_hash.decode()
if genesis_hash != self.coin.GENESIS_HASH:
raise self.DBError(f'DB genesis hash {genesis_hash} does not '
f'match coin {self.coin.GENESIS_HASH}')
self.db_height = state['height']
self.db_tx_count = state['tx_count']
self.db_tip = state['tip']
self.utxo_flush_count = state['utxo_flush_count']
self.wall_time = state['wall_time']
self.first_sync = state['first_sync']
# These are our state as we move ahead of DB state
self.fs_height = self.db_height
self.fs_tx_count = self.db_tx_count
self.last_flush_tx_count = self.fs_tx_count
# Upgrade DB
if self.db_version != max(self.DB_VERSIONS):
self.upgrade_db()
# Log some stats
self.logger.info(f'UTXO DB version: {self.db_version:d}')
self.logger.info(f'coin: {self.coin.NAME}')
self.logger.info(f'network: {self.coin.NET}')
self.logger.info(f'height: {self.db_height:,d}')
self.logger.info(f'tip: {hash_to_hex_str(self.db_tip)}')
self.logger.info(f'tx count: {self.db_tx_count:,d}')
if self.utxo_db.for_sync:
self.logger.info(f'flushing DB cache at {self.env.cache_MB:,d} MB')
if self.first_sync:
self.logger.info(
f'sync time so far: {util.formatted_time(self.wall_time)}')
def upgrade_db(self):
self.logger.info(f'UTXO DB version: {self.db_version}')
self.logger.info('Upgrading your DB; this can take some time...')
def upgrade_u_prefix(prefix):
count = 0
with self.utxo_db.write_batch() as batch:
batch_delete = batch.delete
batch_put = batch.put
# Key: b'u' + address_hashX + tx_idx + tx_num
for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
if len(db_key) == 21:
return
break
if self.db_version == 6:
for db_key, db_value in self.utxo_db.iterator(
prefix=prefix):
count += 1
batch_delete(db_key)
batch_put(db_key[:14] + b'\0\0' + db_key[14:] + b'\0',
db_value)
else:
for db_key, db_value in self.utxo_db.iterator(
prefix=prefix):
count += 1
batch_delete(db_key)
batch_put(db_key + b'\0', db_value)
return count
last = time.time()
count = 0
for cursor in range(65536):
prefix = b'u' + pack_be_uint16(cursor)
count += upgrade_u_prefix(prefix)
now = time.time()
if now > last + 10:
last = now
self.logger.info(f'DB 1 of 3: {count:,d} entries updated, '
f'{cursor * 100 / 65536:.1f}% complete')
self.logger.info('DB 1 of 3 upgraded successfully')
def upgrade_h_prefix(prefix):
count = 0
with self.utxo_db.write_batch() as batch:
batch_delete = batch.delete
batch_put = batch.put
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
if len(db_key) == 14:
return
break
if self.db_version == 6:
for db_key, db_value in self.utxo_db.iterator(
prefix=prefix):
count += 1
batch_delete(db_key)
batch_put(db_key[:7] + b'\0\0' + db_key[7:] + b'\0',
db_value)
else:
for db_key, db_value in self.utxo_db.iterator(
prefix=prefix):
count += 1
batch_delete(db_key)
batch_put(db_key + b'\0', db_value)
return count
last = time.time()
count = 0
for cursor in range(65536):
prefix = b'h' + pack_be_uint16(cursor)
count += upgrade_h_prefix(prefix)
now = time.time()
if now > last + 10:
last = now
self.logger.info(f'DB 2 of 3: {count:,d} entries updated, '
f'{cursor * 100 / 65536:.1f}% complete')
# Upgrade tx_counts file
size = (self.db_height + 1) * 8
tx_counts = self.tx_counts_file.read(0, size)
if len(tx_counts) == (self.db_height + 1) * 4:
tx_counts = array('I', tx_counts)
tx_counts = array('Q', tx_counts)
self.tx_counts_file.write(0, tx_counts.tobytes())
self.db_version = max(self.DB_VERSIONS)
with self.utxo_db.write_batch() as batch:
self.write_utxo_state(batch)
self.logger.info('DB 2 of 3 upgraded successfully')
def write_utxo_state(self, batch):
'''Write (UTXO) state to the batch.'''
state = {
'genesis': self.coin.GENESIS_HASH,
'height': self.db_height,
'tx_count': self.db_tx_count,
'tip': self.db_tip,
'utxo_flush_count': self.utxo_flush_count,
'wall_time': self.wall_time,
'first_sync': self.first_sync,
'db_version': self.db_version,
}
batch.put(b'state', repr(state).encode())
def set_flush_count(self, count):
self.utxo_flush_count = count
with self.utxo_db.write_batch() as batch:
self.write_utxo_state(batch)
async def all_utxos(self, hashX):
'''Return all UTXOs for an address sorted in no particular order.'''
def read_utxos():
utxos = []
utxos_append = utxos.append
txnum_padding = bytes(8 - TXNUM_LEN)
# Key: b'u' + address_hashX + txout_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
prefix = b'u' + hashX
for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
txout_idx, = unpack_le_uint32(db_key[-TXNUM_LEN -
4:-TXNUM_LEN])
tx_num, = unpack_le_uint64(db_key[-TXNUM_LEN:] + txnum_padding)
value, = unpack_le_uint64(db_value)
tx_hash, height = self.fs_tx_hash(tx_num)
utxos_append(UTXO(tx_num, txout_idx, tx_hash, height, value))
return utxos
while True:
utxos = await run_in_thread(read_utxos)
if all(utxo.tx_hash is not None for utxo in utxos):
return utxos
self.logger.warning(f'all_utxos: tx hash not '
f'found (reorg?), retrying...')
await sleep(0.25)
async def lookup_utxos(self, prevouts):
'''For each prevout, lookup it up in the DB and return a (hashX,
value) pair or None if not found.
Used by the mempool code.
'''
def lookup_hashXs():
'''Return (hashX, suffix) pairs, or None if not found,
for each prevout.
'''
def lookup_hashX(tx_hash, tx_idx):
idx_packed = pack_le_uint32(tx_idx)
txnum_padding = bytes(8 - TXNUM_LEN)
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
# Value: hashX
prefix = b'h' + tx_hash[:COMP_TXID_LEN] + idx_packed
# Find which entry, if any, the TX_HASH matches.
for db_key, hashX in self.utxo_db.iterator(prefix=prefix):
tx_num_packed = db_key[-TXNUM_LEN:]
tx_num, = unpack_le_uint64(tx_num_packed + txnum_padding)
hash, _height = self.fs_tx_hash(tx_num)
if hash == tx_hash:
return hashX, idx_packed + tx_num_packed
return None, None
return [lookup_hashX(*prevout) for prevout in prevouts]
def lookup_utxos(hashX_pairs):
def lookup_utxo(hashX, suffix):
if not hashX:
# This can happen when the daemon is a block ahead
# of us and has mempool txs spending outputs from
# that new block
return None
# Key: b'u' + address_hashX + tx_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
key = b'u' + hashX + suffix
db_value = self.utxo_db.get(key)
if not db_value:
# This can happen if the DB was updated between
# getting the hashXs and getting the UTXOs
return None
value, = unpack_le_uint64(db_value)
return hashX, value
return [lookup_utxo(*hashX_pair) for hashX_pair in hashX_pairs]
hashX_pairs = await run_in_thread(lookup_hashXs)
return await run_in_thread(lookup_utxos, hashX_pairs)
def __init__(self, env: 'Env'):
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.env = env
self.coin = env.coin
# Setup block header size handlers
if self.coin.STATIC_BLOCK_HEADERS:
self.header_offset = self.coin.static_header_offset
self.header_len = self.coin.static_header_len
else:
self.header_offset = self.dynamic_header_offset
self.header_len = self.dynamic_header_len
self.logger.info(f'switching current directory to {env.db_dir}')
os.chdir(env.db_dir)
self.db_class = db_class(self.env.db_engine)
self.history = History()
self.eventlog = Eventlog()
self.unflushed_hashYs = defaultdict(set) # {blockHash => [hashY_topic, ]}, for reorg_chain
self.hashY_db = None
# Key: b'u' + address_hashX + txout_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer (in satoshis)
# "at address, at outpoint, there is a UTXO of value v"
# ---
# Key: b'h' + compressed_tx_hash + txout_idx + tx_num
# Value: hashX
# "some outpoint created a UTXO at address"
# ---
# Key: b'U' + block_height
# Value: byte-concat list of (hashX + tx_num + value_sats)
# "undo data: list of UTXOs spent at block height"
self.utxo_db = None
self.utxo_flush_count = 0
self.fs_height = -1
self.fs_tx_count = 0
self.db_height = -1
self.db_tx_count = 0
self.db_tip = None # type: Optional[bytes]
self.tx_counts = None
self.last_flush = time.time()
self.last_flush_tx_count = 0
self.wall_time = 0
self.first_sync = True
self.db_version = -1
self.logger.info(f'using {self.env.db_engine} for DB backend')
# Header merkle cache
self.merkle = Merkle()
self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes)
# on-disk: raw block headers in chain order
self.headers_file = util.LogicalFile('meta/headers', 2, 16000000)
# on-disk: cumulative number of txs at the end of height N
self.tx_counts_file = util.LogicalFile('meta/txcounts', 2, 2000000)
# on-disk: 32 byte txids in chain order, allows (tx_num -> txid) map
self.hashes_file = util.LogicalFile('meta/hashes', 4, 16000000)
if not self.coin.STATIC_BLOCK_HEADERS:
self.headers_offsets_file = util.LogicalFile(
'meta/headers_offsets', 2, 16000000)
class DB:
'''Simple wrapper of the backend database for querying.
Performs no DB update, though the DB will be cleaned on opening if
it was shutdown uncleanly.
'''
DB_VERSIONS = [8]
class DBError(Exception):
'''Raised on general DB errors generally indicating corruption.'''
def __init__(self, env):
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.env = env
self.coin = env.coin
self.logger.info(f'switching current directory to {env.db_dir}')
os.chdir(env.db_dir)
self.db_class = db_class(self.env.db_engine)
self.history = History()
self.utxo_db = None
self.state = None
self.last_flush_state = None
self.fs_height = -1
self.fs_tx_count = 0
self.tx_counts = None
self.logger.info(f'using {self.env.db_engine} for DB backend')
# Header merkle cache
self.merkle = Merkle()
self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes)
self.headers_file = util.LogicalFile('meta/headers', 2, 16000000)
self.tx_counts_file = util.LogicalFile('meta/txcounts', 2, 2000000)
self.hashes_file = util.LogicalFile('meta/hashes', 4, 16000000)
async def _read_tx_counts(self):
if self.tx_counts is not None:
return
# tx_counts[N] has the cumulative number of txs at the end of
# height N. So tx_counts[0] is 1 - the genesis coinbase
size = (self.state.height + 1) * 8
tx_counts = self.tx_counts_file.read(0, size)
assert len(tx_counts) == size
self.tx_counts = array('Q', tx_counts)
if self.tx_counts:
assert self.state.tx_count == self.tx_counts[-1]
else:
assert self.state.tx_count == 0
async def _open_dbs(self, for_sync, compacting):
assert self.utxo_db is None
# First UTXO DB
self.utxo_db = self.db_class('utxo', for_sync)
if self.utxo_db.is_new:
self.logger.info('created new database')
self.logger.info('creating metadata directory')
os.mkdir('meta')
with util.open_file('COIN', create=True) as f:
f.write(f'ElectrumX databases and metadata for '
f'{self.coin.NAME} {self.coin.NET}'.encode())
else:
self.logger.info(f'opened UTXO DB (for sync: {for_sync})')
self.read_utxo_state()
# Then history DB
self.state.flush_count = self.history.open_db(self.db_class, for_sync,
self.state.flush_count,
compacting)
self.clear_excess_undo_info()
# Read TX counts (requires meta directory)
await self._read_tx_counts()
return self.state
async def open_for_compacting(self):
return await self._open_dbs(True, True)
async def open_for_sync(self):
'''Open the databases to sync to the daemon.
When syncing we want to reserve a lot of open files for the
synchronization. When serving clients we want the open files for
serving network connections.
'''
return await self._open_dbs(True, False)
async def open_for_serving(self):
'''Open the databases for serving. If they are already open they are
closed first.
'''
if self.utxo_db:
self.logger.info('closing DBs to re-open for serving')
self.utxo_db.close()
self.history.close_db()
self.utxo_db = None
return await self._open_dbs(False, False)
# Header merkle cache
async def populate_header_merkle_cache(self):
self.logger.info('populating header merkle cache...')
length = max(1, self.state.height - self.env.reorg_limit)
start = time.monotonic()
await self.header_mc.initialize(length)
elapsed = time.monotonic() - start
self.logger.info(f'header merkle cache populated in {elapsed:.1f}s')
async def header_branch_and_root(self, length, height):
return await self.header_mc.branch_and_root(length, height)
# Flushing
def assert_flushed(self, flush_data):
'''Asserts state is fully flushed.'''
assert flush_data.state.tx_count == self.fs_tx_count == self.state.tx_count
assert flush_data.state.height == self.fs_height == self.state.height
assert flush_data.state.tip == self.state.tip
assert not flush_data.headers
assert not flush_data.block_tx_hashes
assert not flush_data.adds
assert not flush_data.deletes
assert not flush_data.undo_infos
self.history.assert_flushed()
def flush_dbs(self, flush_data, flush_utxos, size_remaining):
'''Flush out cached state. History is always flushed; UTXOs are
flushed if flush_utxos.'''
if flush_data.state.height == self.state.height:
self.assert_flushed(flush_data)
return
start_time = time.time()
# Flush to file system
self.flush_fs(flush_data)
# Then history
self.flush_history()
flush_data.state.flush_count = self.history.flush_count
# Flush state last as it reads the wall time.
if flush_utxos:
self.flush_utxo_db(flush_data)
end_time = time.time()
elapsed = end_time - start_time
flush_interval = end_time - self.last_flush_state.flush_time
flush_data.state.flush_time = end_time
flush_data.state.sync_time += flush_interval
# Update and flush state again so as not to drop the batch commit time
if flush_utxos:
self.state = flush_data.state.copy()
self.write_utxo_state(self.utxo_db)
tx_delta = flush_data.state.tx_count - self.last_flush_state.tx_count
size_delta = flush_data.state.chain_size - self.last_flush_state.chain_size
self.logger.info(
f'flush #{self.history.flush_count:,d} took {elapsed:.1f}s. '
f'Height {flush_data.state.height:,d} '
f'txs: {flush_data.state.tx_count:,d} ({tx_delta:+,d}) '
f'size: {flush_data.state.chain_size:,d} ({size_delta:+,d})')
# Catch-up stats
if self.utxo_db.for_sync:
size_per_sec_gen = flush_data.state.chain_size / (
flush_data.state.sync_time + 0.01)
size_per_sec_last = size_delta / (flush_interval + 0.01)
eta = size_remaining / (size_per_sec_last + 0.01)
self.logger.info(
f'MB/sec since genesis: {size_per_sec_gen / 1_000_000:.2f}, '
f'since last flush: {size_per_sec_last / 1_000_000:.2f}')
self.logger.info(
f'sync time: {formatted_time(flush_data.state.sync_time)} '
f'ETA: {formatted_time(eta)}')
self.last_flush_state = flush_data.state.copy()
def flush_fs(self, flush_data):
'''Write headers, tx counts and block tx hashes to the filesystem.
The first height to write is self.fs_height + 1. The FS
metadata is all append-only, so in a crash we just pick up
again from the height stored in the DB.
'''
prior_tx_count = (self.tx_counts[self.fs_height]
if self.fs_height >= 0 else 0)
assert len(flush_data.block_tx_hashes) == len(flush_data.headers)
assert flush_data.state.height == self.fs_height + len(
flush_data.headers)
assert flush_data.state.tx_count == (self.tx_counts[-1]
if self.tx_counts else 0)
assert len(self.tx_counts) == flush_data.state.height + 1
hashes = b''.join(flush_data.block_tx_hashes)
flush_data.block_tx_hashes.clear()
assert len(hashes) % 32 == 0
assert len(hashes) // 32 == flush_data.state.tx_count - prior_tx_count
# Write the headers, tx counts, and tx hashes
height_start = self.fs_height + 1
offset = height_start * 80
self.headers_file.write(offset, b''.join(flush_data.headers))
flush_data.headers.clear()
offset = height_start * self.tx_counts.itemsize
self.tx_counts_file.write(offset,
self.tx_counts[height_start:].tobytes())
offset = prior_tx_count * 32
self.hashes_file.write(offset, hashes)
self.fs_height = flush_data.state.height
self.fs_tx_count = flush_data.state.tx_count
def flush_history(self):
self.history.flush()
def flush_utxo_db(self, flush_data):
'''Flush the cached DB writes and UTXO set to the batch.'''
# Care is needed because the writes generated by flushing the
# UTXO state may have keys in common with our write cache or
# may be in the DB already.
start_time = time.monotonic()
add_count = len(flush_data.adds)
spend_count = len(flush_data.deletes) // 2
with self.utxo_db.write_batch() as batch:
# Spends
batch_delete = batch.delete
for key in sorted(flush_data.deletes):
batch_delete(key)
flush_data.deletes.clear()
# New UTXOs
batch_put = batch.put
for key, value in flush_data.adds.items():
# suffix = tx_idx + tx_num
hashX = value[:-13]
suffix = key[-4:] + value[-13:-8]
batch_put(b'h' + key[:4] + suffix, hashX)
batch_put(b'u' + hashX + suffix, value[-8:])
flush_data.adds.clear()
# New undo information
self.flush_undo_infos(batch_put, flush_data.undo_infos)
flush_data.undo_infos.clear()
if self.utxo_db.for_sync:
block_count = flush_data.state.height - self.state.height
tx_count = flush_data.state.tx_count - self.state.tx_count
size = (flush_data.state.chain_size -
self.state.chain_size) / 1_000_000_000
elapsed = time.monotonic() - start_time
self.logger.info(
f'flushed {block_count:,d} blocks size {size:.1f} GB with '
f'{tx_count:,d} txs, {add_count:,d} UTXO adds, '
f'{spend_count:,d} spends in '
f'{elapsed:.1f}s, committing...')
self.state = flush_data.state.copy()
self.write_utxo_state(batch)
def flush_backup(self, flush_data, touched):
'''Like flush_dbs() but when backing up. All UTXOs are flushed.'''
assert not flush_data.headers
assert not flush_data.block_tx_hashes
assert flush_data.state.height < self.state.height
self.history.assert_flushed()
start_time = time.time()
self.backup_fs(flush_data.state.height, flush_data.state.tx_count)
self.history.backup(touched, flush_data.state.tx_count)
self.flush_utxo_db(flush_data)
elapsed = time.time() - start_time
tx_delta = flush_data.state.tx_count - self.last_flush_state.tx_count
size_delta = flush_data.state.chain_size - self.last_flush_state.chain_size
self.logger.info(
f'backup flush #{self.history.flush_count:,d} took '
f'{elapsed:.1f}s. Height {flush_data.state.height:,d} '
f'txs: {flush_data.state.tx_count:,d} ({tx_delta:+,d}) '
f'size: {flush_data.state.chain_size:,d} ({size_delta:+,d})')
self.last_flush_state = flush_data.state.copy()
def backup_fs(self, height, tx_count):
'''Back up during a reorg. This just updates our pointers.'''
self.fs_height = height
self.fs_tx_count = tx_count
# Truncate header_mc: header count is 1 more than the height.
self.header_mc.truncate(height + 1)
async def raw_header(self, height):
'''Return the binary header at the given height.'''
header, n = await self.read_headers(height, 1)
if n != 1:
raise IndexError(f'height {height:,d} out of range')
return header
async def read_headers(self, start_height, count):
'''Requires start_height >= 0, count >= 0. Reads as many headers as are available
starting at start_height up to count. This would be zero if start_height is
beyond state.height, for example.
Returns a (binary, n) pair where binary is the concatenated binary headers, and n
is the count of headers returned.
'''
if start_height < 0 or count < 0:
raise self.DBError(f'{count:,d} headers starting at '
f'{start_height:,d} not on disk')
def read_headers():
# Read some from disk
disk_count = max(0, min(count,
self.state.height + 1 - start_height))
if disk_count:
offset = start_height * 80
size = disk_count * 80
return self.headers_file.read(offset, size), disk_count
return b'', 0
return await run_in_thread(read_headers)
def fs_tx_hash(self, tx_num):
'''Return a pair (tx_hash, tx_height) for the given tx number.
If the tx_height is not on disk, returns (None, tx_height).'''
tx_height = bisect_right(self.tx_counts, tx_num)
if tx_height > self.state.height:
tx_hash = None
else:
tx_hash = self.hashes_file.read(tx_num * 32, 32)
return tx_hash, tx_height
def fs_tx_hashes_at_blockheight(self, block_height):
'''Return a list of tx_hashes at given block height,
in the same order as in the block.
'''
if block_height > self.state.height:
raise self.DBError(
f'block {block_height:,d} not on disk (>{self.state.height:,d})'
)
assert block_height >= 0
if block_height > 0:
first_tx_num = self.tx_counts[block_height - 1]
else:
first_tx_num = 0
num_txs_in_block = self.tx_counts[block_height] - first_tx_num
tx_hashes = self.hashes_file.read(first_tx_num * 32,
num_txs_in_block * 32)
assert num_txs_in_block == len(tx_hashes) // 32
return [
tx_hashes[idx * 32:(idx + 1) * 32]
for idx in range(num_txs_in_block)
]
async def tx_hashes_at_blockheight(self, block_height):
return await run_in_thread(self.fs_tx_hashes_at_blockheight,
block_height)
async def fs_block_hashes(self, height, count):
headers_concat, headers_count = await self.read_headers(height, count)
if headers_count != count:
raise self.DBError(
f'only got {headers_count:,d} headers starting at {height:,d}, '
f'not {count:,d}')
offset = 0
hlen = 80
headers = []
for _ in range(count):
headers.append(headers_concat[offset:offset + hlen])
offset += hlen
return [self.coin.header_hash(header) for header in headers]
async def limited_history(self, hashX, *, limit=1000):
'''Return an unpruned, sorted list of (tx_hash, height) tuples of
confirmed transactions that touched the address, earliest in
the blockchain first. Includes both spending and receiving
transactions. By default returns at most 1000 entries. Set
limit to None to get them all.
'''
def read_history():
tx_nums = list(self.history.get_txnums(hashX, limit))
fs_tx_hash = self.fs_tx_hash
return [fs_tx_hash(tx_num) for tx_num in tx_nums]
while True:
history = await run_in_thread(read_history)
if all(hash is not None for hash, height in history):
return history
self.logger.warning(
'limited_history: tx hash not found (reorg?), retrying...')
await sleep(0.25)
# -- Undo information
def min_undo_height(self, max_height):
'''Returns a height from which we should store undo info.'''
return max_height - self.env.reorg_limit + 1
def undo_key(self, height):
'''DB key for undo information at the given height.'''
return b'U' + pack_be_uint32(height)
def read_undo_info(self, height):
'''Read undo information from a file for the current height.'''
return self.utxo_db.get(self.undo_key(height))
def flush_undo_infos(self, batch_put, undo_infos):
'''undo_infos is a list of (undo_info, height) pairs.'''
for undo_info, height in undo_infos:
batch_put(self.undo_key(height), b''.join(undo_info))
def clear_excess_undo_info(self):
'''Clear excess undo info. Only most recent N are kept.'''
prefix = b'U'
min_height = self.min_undo_height(self.state.height)
keys = []
for key, _hist in self.utxo_db.iterator(prefix=prefix):
height, = unpack_be_uint32(key[-4:])
if height >= min_height:
break
keys.append(key)
if keys:
with self.utxo_db.write_batch() as batch:
for key in keys:
batch.delete(key)
self.logger.info(f'deleted {len(keys):,d} stale undo entries')
# -- UTXO database
def read_utxo_state(self):
now = time.time()
state = self.utxo_db.get(b'state')
if not state:
state = ChainState(height=-1,
tx_count=0,
chain_size=0,
tip=bytes(32),
flush_count=0,
sync_time=0,
flush_time=now,
first_sync=True,
db_version=max(self.DB_VERSIONS))
else:
state = ast.literal_eval(state.decode())
if not isinstance(state, dict):
raise self.DBError('failed reading state from DB')
if state['genesis'] != self.coin.GENESIS_HASH:
raise self.DBError(
f'DB genesis hash {state["genesis"]} does not match '
f'coin {self.coin.GENESIS_HASH}')
state = ChainState(
height=state['height'],
tx_count=state['tx_count'],
chain_size=state.get('chain_size', 0),
tip=state['tip'],
flush_count=state['utxo_flush_count'],
sync_time=state['wall_time'],
flush_time=now,
first_sync=state['first_sync'],
db_version=state['db_version'],
)
self.state = state
self.last_flush_state = state.copy()
if state.db_version not in self.DB_VERSIONS:
raise self.DBError(
f'your UTXO DB version is {state.db_version} but this '
f'software only handles versions {self.DB_VERSIONS}')
# These are as we flush data to disk ahead of DB state
self.fs_height = state.height
self.fs_tx_count = state.tx_count
# Log some stats
self.logger.info(f'UTXO DB version: {state.db_version:d}')
self.logger.info(f'coin: {self.coin.NAME}')
self.logger.info(f'network: {self.coin.NET}')
self.logger.info(f'height: {state.height:,d}')
self.logger.info(f'tip: {hash_to_hex_str(state.tip)}')
self.logger.info(f'tx count: {state.tx_count:,d}')
self.logger.info(f'chain size: {state.chain_size // 1_000_000_000} GB '
f'({state.chain_size:,d} bytes)')
if self.utxo_db.for_sync:
self.logger.info(f'flushing DB cache at {self.env.cache_MB:,d} MB')
if self.state.first_sync:
self.logger.info(
f'sync time so far: {util.formatted_time(state.sync_time)}')
def write_utxo_state(self, batch):
'''Write (UTXO) state to the batch.'''
state = {
'genesis': self.coin.GENESIS_HASH,
'height': self.state.height,
'tx_count': self.state.tx_count,
'chain_size': self.state.chain_size,
'tip': self.state.tip,
'utxo_flush_count': self.state.flush_count,
'wall_time': self.state.sync_time,
'first_sync': self.state.first_sync,
'db_version': self.state.db_version,
}
batch.put(b'state', repr(state).encode())
def set_flush_count(self, count):
self.state.flush_count = count
self.write_utxo_state(self.utxo_db)
async def all_utxos(self, hashX):
'''Return all UTXOs for an address sorted in no particular order.'''
def read_utxos():
utxos = []
utxos_append = utxos.append
# Key: b'u' + address_hashX + tx_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
prefix = b'u' + hashX
for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
tx_pos, = unpack_le_uint32(db_key[-9:-5])
tx_num, = unpack_le_uint64(db_key[-5:] + bytes(3))
value, = unpack_le_uint64(db_value)
tx_hash, height = self.fs_tx_hash(tx_num)
utxos_append(UTXO(tx_num, tx_pos, tx_hash, height, value))
return utxos
while True:
utxos = await run_in_thread(read_utxos)
if all(utxo.tx_hash is not None for utxo in utxos):
return utxos
self.logger.warning(
'all_utxos: tx hash not found (reorg?), retrying...')
await sleep(0.25)
async def lookup_utxos(self, prevouts):
'''For each prevout, lookup it up in the DB and return a (hashX,
value) pair or None if not found.
Used by the mempool code.
'''
def lookup_hashXs():
'''Return (hashX, suffix) pairs, or None if not found,
for each prevout.
'''
def lookup_hashX(tx_hash, tx_idx):
idx_packed = pack_le_uint32(tx_idx)
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
# Value: hashX
prefix = b'h' + tx_hash[:4] + idx_packed
# Find which entry, if any, the TX_HASH matches.
for db_key, hashX in self.utxo_db.iterator(prefix=prefix):
tx_num_packed = db_key[-5:]
tx_num, = unpack_le_uint64(tx_num_packed + bytes(3))
fs_hash, _height = self.fs_tx_hash(tx_num)
if fs_hash == tx_hash:
return hashX, idx_packed + tx_num_packed
return None, None
return [lookup_hashX(*prevout) for prevout in prevouts]
def lookup_utxos(hashX_pairs):
def lookup_utxo(hashX, suffix):
if not hashX:
# This can happen when the daemon is a block ahead
# of us and has mempool txs spending outputs from
# that new block
return None
# Key: b'u' + address_hashX + tx_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
key = b'u' + hashX + suffix
db_value = self.utxo_db.get(key)
if not db_value:
# This can happen if the DB was updated between
# getting the hashXs and getting the UTXOs
return None
value, = unpack_le_uint64(db_value)
return hashX, value
return [lookup_utxo(*hashX_pair) for hashX_pair in hashX_pairs]
hashX_pairs = await run_in_thread(lookup_hashXs)
return await run_in_thread(lookup_utxos, hashX_pairs)
class BlockProcessor(electrumx.server.db.DB):
'''Process blocks and update the DB state to match.
Employ a prefetcher to prefetch blocks in batches for processing.
Coordinate backing up in case of chain reorganisations.
'''
def __init__(self, env, controller, daemon):
super().__init__(env)
# An incomplete compaction needs to be cancelled otherwise
# restarting it will corrupt the history
self.history.cancel_compaction()
self.daemon = daemon
self.controller = controller
# These are our state as we move ahead of DB state
self.fs_height = self.db_height
self.fs_tx_count = self.db_tx_count
self.height = self.db_height
self.tip = self.db_tip
self.tx_count = self.db_tx_count
self.caught_up_event = asyncio.Event()
self.task_queue = asyncio.Queue()
# Meta
self.cache_MB = env.cache_MB
self.next_cache_check = 0
self.last_flush = time.time()
self.last_flush_tx_count = self.tx_count
self.touched = set()
# Header merkle cache
self.merkle = Merkle()
self.header_mc = None
# Caches of unflushed items.
self.headers = []
self.tx_hashes = []
self.undo_infos = []
# UTXO cache
self.utxo_cache = {}
self.db_deletes = []
self.prefetcher = Prefetcher(self)
if self.utxo_db.for_sync:
self.logger.info('flushing DB cache at {:,d} MB'.format(
self.cache_MB))
def add_task(self, task):
'''Add the task to our task queue.'''
self.task_queue.put_nowait(task)
def on_prefetched_blocks(self, blocks, first):
'''Called by the prefetcher when it has prefetched some blocks.'''
self.add_task(partial(self.check_and_advance_blocks, blocks, first))
def on_prefetcher_first_caught_up(self):
'''Called by the prefetcher when it first catches up.'''
self.add_task(self.first_caught_up)
async def main_loop(self):
'''Main loop for block processing.'''
self.controller.create_task(self.prefetcher.main_loop())
await self.prefetcher.reset_height()
while True:
task = await self.task_queue.get()
await task()
def shutdown(self, executor):
'''Shutdown cleanly and flush to disk.'''
# First stut down the executor; it may be processing a block.
# Then we can flush anything remaining to disk.
executor.shutdown()
if self.height != self.db_height:
self.logger.info('flushing state to DB for a clean shutdown...')
self.flush(True)
async def first_caught_up(self):
'''Called when first caught up to daemon after starting.'''
# Flush everything with updated first_sync->False state.
self.first_sync = False
await self.controller.run_in_executor(self.flush, True)
if self.utxo_db.for_sync:
self.logger.info(f'{electrumx.version} synced to '
f'height {self.height:,d}')
self.open_dbs()
self.logger.info(f'caught up to height {self.height:,d}')
length = max(1, self.height - self.env.reorg_limit)
self.header_mc = MerkleCache(self.merkle, HeaderSource(self), length)
self.logger.info('populated header merkle cache')
# Reorgs use header_mc so safest to set this after initializing it
self.caught_up_event.set()
async def check_and_advance_blocks(self, raw_blocks, first):
'''Process the list of raw blocks passed. Detects and handles
reorgs.
'''
self.prefetcher.processing_blocks(raw_blocks)
if first != self.height + 1:
# If we prefetched two sets of blocks and the first caused
# a reorg this will happen when we try to process the
# second. It should be very rare.
self.logger.warning('ignoring {:,d} blocks starting height {:,d}, '
'expected {:,d}'.format(
len(raw_blocks), first, self.height + 1))
return
blocks = [
self.coin.block(raw_block, first + n)
for n, raw_block in enumerate(raw_blocks)
]
headers = [block.header for block in blocks]
hprevs = [self.coin.header_prevhash(h) for h in headers]
chain = [self.tip] + [self.coin.header_hash(h) for h in headers[:-1]]
if hprevs == chain:
start = time.time()
await self.controller.run_in_executor(self.advance_blocks, blocks)
if not self.first_sync:
s = '' if len(blocks) == 1 else 's'
self.logger.info('processed {:,d} block{} in {:.1f}s'.format(
len(blocks), s,
time.time() - start))
self.controller.mempool.on_new_block(self.touched)
self.touched.clear()
elif hprevs[0] != chain[0]:
await self.reorg_chain()
else:
# It is probably possible but extremely rare that what
# bitcoind returns doesn't form a chain because it
# reorg-ed the chain as it was processing the batched
# block hash requests. Should this happen it's simplest
# just to reset the prefetcher and try again.
self.logger.warning('daemon blocks do not form a chain; '
'resetting the prefetcher')
await self.prefetcher.reset_height()
def force_chain_reorg(self, count):
'''Force a reorg of the given number of blocks.
Returns True if a reorg is queued, false if not caught up.
'''
if self.caught_up_event.is_set():
self.add_task(partial(self.reorg_chain, count=count))
return True
return False
async def reorg_chain(self, count=None):
'''Handle a chain reorganisation.
Count is the number of blocks to simulate a reorg, or None for
a real reorg.'''
if count is None:
self.logger.info('chain reorg detected')
else:
self.logger.info('faking a reorg of {:,d} blocks'.format(count))
await self.controller.run_in_executor(self.flush, True)
hashes = await self.reorg_hashes(count)
# Reverse and convert to hex strings.
hashes = [hash_to_hex_str(hash) for hash in reversed(hashes)]
for hex_hashes in chunks(hashes, 50):
blocks = await self.daemon.raw_blocks(hex_hashes)
await self.controller.run_in_executor(self.backup_blocks, blocks)
# Truncate header_mc: header count is 1 more than the height
self.header_mc.truncate(self.height + 1)
await self.prefetcher.reset_height()
async def reorg_hashes(self, count):
'''Return the list of hashes to back up beacuse of a reorg.
The hashes are returned in order of increasing height.'''
def diff_pos(hashes1, hashes2):
'''Returns the index of the first difference in the hash lists.
If both lists match returns their length.'''
for n, (hash1, hash2) in enumerate(zip(hashes1, hashes2)):
if hash1 != hash2:
return n
return len(hashes)
if count is None:
# A real reorg
start = self.height - 1
count = 1
while start > 0:
hashes = self.fs_block_hashes(start, count)
hex_hashes = [hash_to_hex_str(hash) for hash in hashes]
d_hex_hashes = await self.daemon.block_hex_hashes(start, count)
n = diff_pos(hex_hashes, d_hex_hashes)
if n > 0:
start += n
break
count = min(count * 2, start)
start -= count
count = (self.height - start) + 1
else:
start = (self.height - count) + 1
s = '' if count == 1 else 's'
self.logger.info('chain was reorganised replacing {:,d} block{} at '
'heights {:,d}-{:,d}'.format(count, s, start,
start + count - 1))
return self.fs_block_hashes(start, count)
def flush_state(self, batch):
'''Flush chain state to the batch.'''
now = time.time()
self.wall_time += now - self.last_flush
self.last_flush = now
self.last_flush_tx_count = self.tx_count
self.write_utxo_state(batch)
def assert_flushed(self):
'''Asserts state is fully flushed.'''
assert self.tx_count == self.fs_tx_count == self.db_tx_count
assert self.height == self.fs_height == self.db_height
assert not self.undo_infos
assert not self.utxo_cache
assert not self.db_deletes
self.history.assert_flushed()
def flush(self, flush_utxos=False):
'''Flush out cached state.
History is always flushed. UTXOs are flushed if flush_utxos.'''
if self.height == self.db_height:
self.assert_flushed()
return
flush_start = time.time()
last_flush = self.last_flush
tx_diff = self.tx_count - self.last_flush_tx_count
# Flush to file system
self.fs_flush()
fs_end = time.time()
if self.utxo_db.for_sync:
self.logger.info('flushed to FS in {:.1f}s'.format(fs_end -
flush_start))
# History next - it's fast and frees memory
hashX_count = self.history.flush()
if self.utxo_db.for_sync:
self.logger.info(
'flushed history in {:.1f}s for {:,d} addrs'.format(
time.time() - fs_end, hashX_count))
# Flush state last as it reads the wall time.
with self.utxo_db.write_batch() as batch:
if flush_utxos:
self.flush_utxos(batch)
self.flush_state(batch)
# Update and put the wall time again - otherwise we drop the
# time it took to commit the batch
self.flush_state(self.utxo_db)
self.logger.info(
'flush #{:,d} took {:.1f}s. Height {:,d} txs: {:,d}'.format(
self.history.flush_count, self.last_flush - flush_start,
self.height, self.tx_count))
# Catch-up stats
if self.utxo_db.for_sync:
tx_per_sec = int(self.tx_count / self.wall_time)
this_tx_per_sec = 1 + int(tx_diff / (self.last_flush - last_flush))
self.logger.info('tx/sec since genesis: {:,d}, '
'since last flush: {:,d}'.format(
tx_per_sec, this_tx_per_sec))
daemon_height = self.daemon.cached_height()
if self.height > self.coin.TX_COUNT_HEIGHT:
tx_est = (daemon_height - self.height) * self.coin.TX_PER_BLOCK
else:
tx_est = ((daemon_height - self.coin.TX_COUNT_HEIGHT) *
self.coin.TX_PER_BLOCK +
(self.coin.TX_COUNT - self.tx_count))
# Damp the enthusiasm
realism = 2.0 - 0.9 * self.height / self.coin.TX_COUNT_HEIGHT
tx_est *= max(realism, 1.0)
self.logger.info('sync time: {} ETA: {}'.format(
formatted_time(self.wall_time),
formatted_time(tx_est / this_tx_per_sec)))
def fs_flush(self):
'''Flush the things stored on the filesystem.'''
assert self.fs_height + len(self.headers) == self.height
assert self.tx_count == self.tx_counts[-1] if self.tx_counts else 0
self.fs_update(self.fs_height, self.headers, self.tx_hashes)
self.fs_height = self.height
self.fs_tx_count = self.tx_count
self.tx_hashes = []
self.headers = []
def backup_flush(self):
'''Like flush() but when backing up. All UTXOs are flushed.
hashXs - sequence of hashXs which were touched by backing
up. Searched for history entries to remove after the backup
height.
'''
assert self.height < self.db_height
self.history.assert_flushed()
flush_start = time.time()
# Backup FS (just move the pointers back)
self.fs_height = self.height
self.fs_tx_count = self.tx_count
assert not self.headers
assert not self.tx_hashes
# Backup history. self.touched can include other addresses
# which is harmless, but remove None.
self.touched.discard(None)
nremoves = self.history.backup(self.touched, self.tx_count)
self.logger.info(
'backing up removed {:,d} history entries'.format(nremoves))
with self.utxo_db.write_batch() as batch:
# Flush state last as it reads the wall time.
self.flush_utxos(batch)
self.flush_state(batch)
self.logger.info('backup flush #{:,d} took {:.1f}s. '
'Height {:,d} txs: {:,d}'.format(
self.history.flush_count,
self.last_flush - flush_start, self.height,
self.tx_count))
def check_cache_size(self):
'''Flush a cache if it gets too big.'''
# Good average estimates based on traversal of subobjects and
# requesting size from Python (see deep_getsizeof).
one_MB = 1000 * 1000
utxo_cache_size = len(self.utxo_cache) * 205
db_deletes_size = len(self.db_deletes) * 57
hist_cache_size = self.history.unflushed_memsize()
# Roughly ntxs * 32 + nblocks * 42
tx_hash_size = ((self.tx_count - self.fs_tx_count) * 32 +
(self.height - self.fs_height) * 42)
utxo_MB = (db_deletes_size + utxo_cache_size) // one_MB
hist_MB = (hist_cache_size + tx_hash_size) // one_MB
self.logger.info('our height: {:,d} daemon: {:,d} '
'UTXOs {:,d}MB hist {:,d}MB'.format(
self.height, self.daemon.cached_height(), utxo_MB,
hist_MB))
# Flush history if it takes up over 20% of cache memory.
# Flush UTXOs once they take up 80% of cache memory.
if utxo_MB + hist_MB >= self.cache_MB or hist_MB >= self.cache_MB // 5:
self.flush(utxo_MB >= self.cache_MB * 4 // 5)
def advance_blocks(self, blocks):
'''Synchronously advance the blocks.
It is already verified they correctly connect onto our tip.
'''
min_height = self.min_undo_height(self.daemon.cached_height())
height = self.height
for block in blocks:
height += 1
undo_info = self.advance_txs(block.transactions)
if height >= min_height:
self.undo_infos.append((undo_info, height))
headers = [block.header for block in blocks]
self.height = height
self.headers.extend(headers)
self.tip = self.coin.header_hash(headers[-1])
# If caught up, flush everything as client queries are
# performed on the DB.
if self.caught_up_event.is_set():
self.flush(True)
else:
if time.time() > self.next_cache_check:
self.check_cache_size()
self.next_cache_check = time.time() + 30
def advance_txs(self, txs):
self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs))
# Use local vars for speed in the loops
undo_info = []
tx_num = self.tx_count
script_hashX = self.coin.hashX_from_script
s_pack = pack
put_utxo = self.utxo_cache.__setitem__
spend_utxo = self.spend_utxo
undo_info_append = undo_info.append
update_touched = self.touched.update
hashXs_by_tx = []
append_hashXs = hashXs_by_tx.append
for tx, tx_hash in txs:
hashXs = []
append_hashX = hashXs.append
tx_numb = s_pack('<I', tx_num)
# Spend the inputs
if not tx.is_coinbase:
for txin in tx.inputs:
cache_value = spend_utxo(txin.prev_hash, txin.prev_idx)
undo_info_append(cache_value)
append_hashX(cache_value[:-12])
# Add the new UTXOs
for idx, txout in enumerate(tx.outputs):
# Get the hashX. Ignore unspendable outputs
hashX = script_hashX(txout.pk_script)
if hashX:
append_hashX(hashX)
put_utxo(tx_hash + s_pack('<H', idx),
hashX + tx_numb + s_pack('<Q', txout.value))
append_hashXs(hashXs)
update_touched(hashXs)
tx_num += 1
self.history.add_unflushed(hashXs_by_tx, self.tx_count)
self.tx_count = tx_num
self.tx_counts.append(tx_num)
return undo_info
def backup_blocks(self, raw_blocks):
'''Backup the raw blocks and flush.
The blocks should be in order of decreasing height, starting at.
self.height. A flush is performed once the blocks are backed up.
'''
self.assert_flushed()
assert self.height >= len(raw_blocks)
coin = self.coin
for raw_block in raw_blocks:
# Check and update self.tip
block = coin.block(raw_block, self.height)
header_hash = coin.header_hash(block.header)
if header_hash != self.tip:
raise ChainError(
'backup block {} not tip {} at height {:,d}'.format(
hash_to_hex_str(header_hash),
hash_to_hex_str(self.tip), self.height))
self.tip = coin.header_prevhash(block.header)
self.backup_txs(block.transactions)
self.height -= 1
self.tx_counts.pop()
self.logger.info('backed up to height {:,d}'.format(self.height))
self.backup_flush()
def backup_txs(self, txs):
# Prevout values, in order down the block (coinbase first if present)
# undo_info is in reverse block order
undo_info = self.read_undo_info(self.height)
if undo_info is None:
raise ChainError(
'no undo information found for height {:,d}'.format(
self.height))
n = len(undo_info)
# Use local vars for speed in the loops
s_pack = pack
put_utxo = self.utxo_cache.__setitem__
spend_utxo = self.spend_utxo
script_hashX = self.coin.hashX_from_script
touched = self.touched
undo_entry_len = 12 + HASHX_LEN
for tx, tx_hash in reversed(txs):
for idx, txout in enumerate(tx.outputs):
# Spend the TX outputs. Be careful with unspendable
# outputs - we didn't save those in the first place.
hashX = script_hashX(txout.pk_script)
if hashX:
cache_value = spend_utxo(tx_hash, idx)
touched.add(cache_value[:-12])
# Restore the inputs
if not tx.is_coinbase:
for txin in reversed(tx.inputs):
n -= undo_entry_len
undo_item = undo_info[n:n + undo_entry_len]
put_utxo(txin.prev_hash + s_pack('<H', txin.prev_idx),
undo_item)
touched.add(undo_item[:-12])
assert n == 0
self.tx_count -= len(txs)
'''An in-memory UTXO cache, representing all changes to UTXO state
since the last DB flush.
We want to store millions of these in memory for optimal
performance during initial sync, because then it is possible to
spend UTXOs without ever going to the database (other than as an
entry in the address history, and there is only one such entry per
TX not per UTXO). So store them in a Python dictionary with
binary keys and values.
Key: TX_HASH + TX_IDX (32 + 2 = 34 bytes)
Value: HASHX + TX_NUM + VALUE (11 + 4 + 8 = 23 bytes)
That's 57 bytes of raw data in-memory. Python dictionary overhead
means each entry actually uses about 205 bytes of memory. So
almost 5 million UTXOs can fit in 1GB of RAM. There are
approximately 42 million UTXOs on bitcoin mainnet at height
433,000.
Semantics:
add: Add it to the cache dictionary.
spend: Remove it if in the cache dictionary. Otherwise it's
been flushed to the DB. Each UTXO is responsible for two
entries in the DB. Mark them for deletion in the next
cache flush.
The UTXO database format has to be able to do two things efficiently:
1. Given an address be able to list its UTXOs and their values
so its balance can be efficiently computed.
2. When processing transactions, for each prevout spent - a (tx_hash,
idx) pair - we have to be able to remove it from the DB. To send
notifications to clients we also need to know any address it paid
to.
To this end we maintain two "tables", one for each point above:
1. Key: b'u' + address_hashX + tx_idx + tx_num
Value: the UTXO value as a 64-bit unsigned integer
2. Key: b'h' + compressed_tx_hash + tx_idx + tx_num
Value: hashX
The compressed tx hash is just the first few bytes of the hash of
the tx in which the UTXO was created. As this is not unique there
will be potential collisions so tx_num is also in the key. When
looking up a UTXO the prefix space of the compressed hash needs to
be searched and resolved if necessary with the tx_num. The
collision rate is low (<0.1%).
'''
def spend_utxo(self, tx_hash, tx_idx):
'''Spend a UTXO and return the 33-byte value.
If the UTXO is not in the cache it must be on disk. We store
all UTXOs so not finding one indicates a logic error or DB
corruption.
'''
# Fast track is it being in the cache
idx_packed = pack('<H', tx_idx)
cache_value = self.utxo_cache.pop(tx_hash + idx_packed, None)
if cache_value:
return cache_value
# Spend it from the DB.
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
# Value: hashX
prefix = b'h' + tx_hash[:4] + idx_packed
candidates = {
db_key: hashX
for db_key, hashX in self.utxo_db.iterator(prefix=prefix)
}
for hdb_key, hashX in candidates.items():
tx_num_packed = hdb_key[-4:]
if len(candidates) > 1:
tx_num, = unpack('<I', tx_num_packed)
hash, height = self.fs_tx_hash(tx_num)
if hash != tx_hash:
assert hash is not None # Should always be found
continue
# Key: b'u' + address_hashX + tx_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
udb_key = b'u' + hashX + hdb_key[-6:]
utxo_value_packed = self.utxo_db.get(udb_key)
if utxo_value_packed:
# Remove both entries for this UTXO
self.db_deletes.append(hdb_key)
self.db_deletes.append(udb_key)
return hashX + tx_num_packed + utxo_value_packed
raise ChainError('UTXO {} / {:,d} not found in "h" table'.format(
hash_to_hex_str(tx_hash), tx_idx))
def flush_utxos(self, batch):
'''Flush the cached DB writes and UTXO set to the batch.'''
# Care is needed because the writes generated by flushing the
# UTXO state may have keys in common with our write cache or
# may be in the DB already.
flush_start = time.time()
delete_count = len(self.db_deletes) // 2
utxo_cache_len = len(self.utxo_cache)
# Spends
batch_delete = batch.delete
for key in sorted(self.db_deletes):
batch_delete(key)
self.db_deletes = []
# New UTXOs
batch_put = batch.put
for cache_key, cache_value in self.utxo_cache.items():
# suffix = tx_idx + tx_num
hashX = cache_value[:-12]
suffix = cache_key[-2:] + cache_value[-12:-8]
batch_put(b'h' + cache_key[:4] + suffix, hashX)
batch_put(b'u' + hashX + suffix, cache_value[-8:])
self.utxo_cache = {}
# New undo information
self.flush_undo_infos(batch_put, self.undo_infos)
self.undo_infos = []
if self.utxo_db.for_sync:
self.logger.info(
'flushed {:,d} blocks with {:,d} txs, {:,d} UTXO '
'adds, {:,d} spends in {:.1f}s, committing...'.format(
self.height - self.db_height,
self.tx_count - self.db_tx_count, utxo_cache_len,
delete_count,
time.time() - flush_start))
self.utxo_flush_count = self.history.flush_count
self.db_tx_count = self.tx_count
self.db_height = self.height
self.db_tip = self.tip
def test_markle_cache_bad():
length = 23
source = Source(length)
cache = MerkleCache(merkle, source, length)
cache.branch_and_root(5, 3)
with pytest.raises(TypeError):
cache.branch_and_root(5.0, 3)
with pytest.raises(TypeError):
cache.branch_and_root(5, 3.0)
with pytest.raises(ValueError):
cache.branch_and_root(0, -1)
with pytest.raises(ValueError):
cache.branch_and_root(3, 3)
