def __dump_tx_weights_in_block_to_file(h: BlockHeight, filepath: str) -> bool:
"""
helper for dump_tx_weights_in_block. write weight of txs in block h to a file in the given path.
returns true only if ALL weights were successfully written to the file.
"""
try:
block: Block = get_block_by_height(h)
except Exception:
logger.error(f"Failed to retrieve block {h} from bitcoind")
return False
executor = ThreadPoolExecutor(max_workers=MAX_WORKERS)
# submit all jobs to the pool
txid_to_future = {
txid: executor.submit(get_tx_weight, txid)
for txid in block["tx"]
}
with open(filepath, mode="w") as f:
# iterate all futures, extract the computed result and dump to the file
for txid, future in txid_to_future.items():
weight = future.result()
if weight is None:
# we give up on the entire block if we fail to get the weight of at least one transaction
# cancel all remaining jobs
for f in txid_to_future.values():
f.cancel()
executor.shutdown(wait=False)
return False
f.write(f"{txid}{TSV_SEPARATOR}{weight}\n")
executor.shutdown(wait=True)
return True
def populate_blocks(first_block: BlockHeight, last_block: BlockHeight) -> None:
for h in range(first_block, last_block + 1):
try:
logger.info(f"Dumping tx weights+feerates for block {h}")
populate_block(h)
except Exception as e:
logger.error(
f"Exception occurred when trying to populate with txs in block {h}: {type(e)}:{str(e)}"
)
def dump_tx_weights_in_block(h: BlockHeight) -> None:
"""
computes the weight of all transactions in block at height h and dump them
to a text file in the DB_FOLDER directory
"""
logger.info(f"Dumping weights for block {h}")
filepath = os.path.join(TX_WEIGHTS_FOLDER, f"block_{h}_tx_weights.tsv")
if os.path.isfile(filepath):
return # this block was already dumped
# use tmp suffix until we finish with that block (in case we crash before we dumped all txs)
filepath_tmp = f"{filepath}.tmp"
success = __dump_tx_weights_in_block_to_file(h=h, filepath=filepath_tmp)
if success:
os.rename(filepath_tmp, filepath)
else:
logger.error(f"Failed to dump weights for transactions of block {h}")
def dump_block_feerates(h: BlockHeight) -> None:
"""
computes the feerate of all transactions in block at height h and dump them
to a text file in the DB_FOLDER directory
"""
logger.info(f"Dumping feerates for block {h}")
filepath = os.path.join(DATA, "feerates_tsv_files", f"block_{h}_feerates.tsv")
if os.path.isfile(filepath):
return # this block was already dumped
# use tmp suffix until we finish with that block (in case we crash before we dumped all txs)
filepath_tmp = f"{filepath}.tmp"
success = __dump_block_feerates_to_file(h=h, filepath=filepath_tmp)
if success:
os.rename(filepath_tmp, filepath)
else:
logger.error(f"Failed to retrieve feerate for a transaction in block {h}")
def parse_estimation_files() -> Dict[int, List[PlotData]]:
"""
read all fee estimation files and prepare the plot data
returns a dictionary from blocks_count (number of blocks used for the estimation)
to a list of 'graphs' (represented by PLOT_DATA)
"""
data = defaultdict(list)
for entry in os.listdir(FEE_ESTIMATIONS_DIR):
match = estimation_sample_file_regex.fullmatch(entry)
if not match:
continue # not an estimation file
num_blocks: int = int(match.group(1))
mode: str = match.group(2)
if num_blocks_to_include is not None and num_blocks not in num_blocks_to_include:
continue
timestamps = []
feerates = []
with open(os.path.join(FEE_ESTIMATIONS_DIR, entry)) as f:
for line in f.readlines():
try:
timestamp_str, feerate_str = line.strip().split(",")
# feerate returned by `estimatesmartfee` is in BTC/kB
feerate_btc_kb = float(feerate_str)
feerate: Feerate = btc_to_sat(
feerate_btc_kb) / BYTE_IN_KBYTE
timestamp = int(timestamp_str)
if MIN_TIMESTAMP is None or MAX_TIMESTAMP is None or MIN_TIMESTAMP <= timestamp <= MAX_TIMESTAMP:
timestamps.append(timestamp)
feerates.append(feerate)
except ValueError:
logger.error(
f"ignoring line in file `{entry}` with unexpected format: `{line}`"
)
data[num_blocks].append(
PlotData(timestamps=timestamps,
feerates=feerates,
label=f"estimatesmartfee(n={num_blocks},mode={mode})"))
return data
def __dump_block_feerates_to_file(h: BlockHeight, filepath: str) -> bool:
"""
helper for dump_block_feerates. write feerates of txs in block h to a file in the given path.
returns true only if ALL feerates were successfully written to the file.
"""
try:
block: Block = get_block_by_height(h)
except Exception:
# TODO change this except. get_block_by_height should either declare
# exactly what it raises, or return None if it fails
logger.error(f"Failed to retrieve block {h} from bitcoind")
return False
executor = ThreadPoolExecutor(max_workers=MAX_WORKERS)
# submit all jobs to the pool
txid_to_future = {
txid: executor.submit(get_tx_feerate, txid)
for txid in block["tx"]
}
with open(filepath, mode="w") as f:
# iterate all futures, extract the computed result and dump to the file
for txid, future in txid_to_future.items():
try:
feerate = future.result()
except Exception:
# we give up on the entire block if we fail to get the feerate of
# at least one transaction
# cancel all remaining jobs
for f in txid_to_future.values():
f.cancel()
executor.shutdown(wait=False)
return False
f.write(f"{txid}{TSV_SEPARATOR}{feerate}\n")
executor.shutdown(wait=True)
return True