async def run_add_block_benchmark(version: int):
verbose: bool = "--verbose" in sys.argv
db_wrapper: DBWrapper = await setup_db("block-store-benchmark.db", version)
# keep track of benchmark total time
all_test_time = 0.0
prev_block = bytes32([0] * 32)
prev_ses_hash = bytes32([0] * 32)
header_hashes = []
try:
block_store = await BlockStore.create(db_wrapper)
block_height = 1
timestamp = uint64(1631794488)
weight = uint128(10)
iters = uint128(123456)
sp_index = uint8(0)
deficit = uint8(0)
sub_slot_iters = uint64(10)
required_iters = uint64(100)
transaction_block_counter = 0
prev_transaction_block = bytes32([0] * 32)
prev_transaction_height = uint32(0)
total_time = 0.0
ses_counter = 0
if verbose:
print("profiling add_full_block", end="")
for height in range(block_height, block_height + NUM_ITERS):
is_transaction = transaction_block_counter == 0
fees = uint64(random.randint(0, 150000))
farmer_coin, pool_coin = rewards(uint32(height))
reward_claims_incorporated = [farmer_coin, pool_coin]
# TODO: increase fidelity by setting these as well
finished_challenge_slot_hashes = None
finished_infused_challenge_slot_hashes = None
finished_reward_slot_hashes = None
sub_epoch_summary_included = None
if ses_counter == 0:
sub_epoch_summary_included = SubEpochSummary(
prev_ses_hash,
rand_hash(),
uint8(random.randint(0,
255)), # num_blocks_overflow: uint8
None, # new_difficulty: Optional[uint64]
None, # new_sub_slot_iters: Optional[uint64]
)
has_pool_pk = random.randint(0, 1)
proof_of_space = ProofOfSpace(
rand_hash(), # challenge
rand_g1() if has_pool_pk else None,
rand_hash() if not has_pool_pk else None,
rand_g1(), # plot_public_key
uint8(32),
rand_bytes(8 * 32),
)
reward_chain_block = RewardChainBlock(
weight,
uint32(height),
iters,
sp_index,
rand_hash(), # pos_ss_cc_challenge_hash
proof_of_space,
None if sp_index == 0 else rand_vdf(),
rand_g2(), # challenge_chain_sp_signature
rand_vdf(), # challenge_chain_ip_vdf
rand_vdf() if sp_index != 0 else None, # reward_chain_sp_vdf
rand_g2(), # reward_chain_sp_signature
rand_vdf(), # reward_chain_ip_vdf
rand_vdf() if deficit < 16 else None,
is_transaction,
)
pool_target = PoolTarget(
rand_hash(), # puzzle_hash
uint32(0), # max_height
)
foliage_block_data = FoliageBlockData(
rand_hash(), # unfinished_reward_block_hash
pool_target,
rand_g2() if has_pool_pk else None, # pool_signature
rand_hash(), # farmer_reward_puzzle_hash
bytes32([0] * 32), # extension_data
)
foliage = Foliage(
prev_block,
rand_hash(), # reward_block_hash
foliage_block_data,
rand_g2(), # foliage_block_data_signature
rand_hash()
if is_transaction else None, # foliage_transaction_block_hash
rand_g2() if is_transaction else
None, # foliage_transaction_block_signature
)
foliage_transaction_block = (
None if not is_transaction else FoliageTransactionBlock(
prev_transaction_block,
timestamp,
rand_hash(), # filter_hash
rand_hash(), # additions_root
rand_hash(), # removals_root
rand_hash(), # transactions_info_hash
))
transactions_info = (
None if not is_transaction else TransactionsInfo(
rand_hash(), # generator_root
rand_hash(), # generator_refs_root
rand_g2(), # aggregated_signature
fees,
uint64(random.randint(0, 12000000000)), # cost
reward_claims_incorporated,
))
full_block = FullBlock(
[], # finished_sub_slots
reward_chain_block,
rand_vdf_proof()
if sp_index > 0 else None, # challenge_chain_sp_proof
rand_vdf_proof(), # challenge_chain_ip_proof
rand_vdf_proof()
if sp_index > 0 else None, # reward_chain_sp_proof
rand_vdf_proof(), # reward_chain_ip_proof
rand_vdf_proof()
if deficit < 4 else None, # infused_challenge_chain_ip_proof
foliage,
foliage_transaction_block,
transactions_info,
None if is_transaction else SerializedProgram.from_bytes(
clvm_generator), # transactions_generator
[], # transactions_generator_ref_list
)
header_hash = full_block.header_hash
record = BlockRecord(
header_hash,
prev_block,
uint32(height),
weight,
iters,
sp_index,
rand_class_group_element(),
None if deficit > 3 else rand_class_group_element(),
rand_hash(), # reward_infusion_new_challenge
rand_hash(), # challenge_block_info_hash
sub_slot_iters,
rand_hash(), # pool_puzzle_hash
rand_hash(), # farmer_puzzle_hash
required_iters,
deficit,
deficit == 16,
prev_transaction_height,
timestamp if is_transaction else None,
prev_transaction_block
if prev_transaction_block != bytes32([0] * 32) else None,
None if fees == 0 else fees,
reward_claims_incorporated,
finished_challenge_slot_hashes,
finished_infused_challenge_slot_hashes,
finished_reward_slot_hashes,
sub_epoch_summary_included,
)
start = time()
await block_store.add_full_block(header_hash, full_block, record,
False)
await block_store.set_in_chain([(header_hash, )])
header_hashes.append(header_hash)
await block_store.set_peak(header_hash)
await db_wrapper.db.commit()
stop = time()
total_time += stop - start
# 19 seconds per block
timestamp = uint64(timestamp + 19)
weight = uint128(weight + 10)
iters = uint128(iters + 123456)
sp_index = uint8((sp_index + 1) % 64)
deficit = uint8((deficit + 3) % 17)
ses_counter = (ses_counter + 1) % 384
prev_block = header_hash
# every 33 blocks is a transaction block
transaction_block_counter = (transaction_block_counter + 1) % 33
if is_transaction:
prev_transaction_block = header_hash
prev_transaction_height = uint32(height)
if ses_counter == 0:
prev_ses_hash = header_hash
if verbose:
print(".", end="")
sys.stdout.flush()
block_height += NUM_ITERS
if verbose:
print("")
print(f"{total_time:0.4f}s, add_full_block")
all_test_time += total_time
total_time = 0.0
if verbose:
print("profiling get_full_block")
random.shuffle(header_hashes)
start = time()
for h in header_hashes:
block = await block_store.get_full_block(h)
assert block.header_hash == h
stop = time()
total_time += stop - start
print(f"{total_time:0.4f}s, get_full_block")
all_test_time += total_time
total_time = 0.0
if verbose:
print("profiling get_full_block_bytes")
start = time()
for h in header_hashes:
block = await block_store.get_full_block_bytes(h)
assert len(block) > 0
stop = time()
total_time += stop - start
print(f"{total_time:0.4f}s, get_full_block_bytes")
all_test_time += total_time
total_time = 0.0
if verbose:
print("profiling get_full_blocks_at")
start = time()
for h in range(1, block_height):
blocks = await block_store.get_full_blocks_at([h])
assert len(blocks) == 1
assert blocks[0].height == h
stop = time()
total_time += stop - start
print(f"{total_time:0.4f}s, get_full_blocks_at")
all_test_time += total_time
total_time = 0.0
if verbose:
print("profiling get_block_records_by_hash")
start = time()
for h in header_hashes:
blocks = await block_store.get_block_records_by_hash([h])
assert len(blocks) == 1
assert blocks[0].header_hash == h
stop = time()
total_time += stop - start
print(f"{total_time:0.4f}s, get_block_records_by_hash")
all_test_time += total_time
total_time = 0.0
if verbose:
print("profiling get_blocks_by_hash")
start = time()
for h in header_hashes:
blocks = await block_store.get_blocks_by_hash([h])
assert len(blocks) == 1
assert blocks[0].header_hash == h
stop = time()
total_time += stop - start
print(f"{total_time:0.4f}s, get_blocks_by_hash")
all_test_time += total_time
total_time = 0.0
if verbose:
print("profiling get_block_record")
start = time()
for h in header_hashes:
blocks = await block_store.get_block_record(h)
assert blocks.header_hash == h
stop = time()
total_time += stop - start
print(f"{total_time:0.4f}s, get_block_record")
all_test_time += total_time
total_time = 0.0
if verbose:
print("profiling get_block_records_in_range")
start = time()
for i in range(100):
h = random.randint(1, block_height - 100)
blocks = await block_store.get_block_records_in_range(h, h + 99)
assert len(blocks) == 100
stop = time()
total_time += stop - start
print(f"{total_time:0.4f}s, get_block_records_in_range")
all_test_time += total_time
total_time = 0.0
if verbose:
print("profiling get_block_records_close_to_peak")
start = time()
blocks, peak = await block_store.get_block_records_close_to_peak(99)
assert len(blocks) == 100
stop = time()
total_time += stop - start
print(f"{total_time:0.4f}s, get_block_records_close_to_peak")
all_test_time += total_time
total_time = 0.0
if verbose:
print("profiling is_fully_compactified")
start = time()
for h in header_hashes:
compactified = await block_store.is_fully_compactified(h)
assert compactified is False
stop = time()
total_time += stop - start
print(f"{total_time:0.4f}s, get_block_record")
all_test_time += total_time
total_time = 0.0
if verbose:
print("profiling get_random_not_compactified")
start = time()
for i in range(1, 5000):
blocks = await block_store.get_random_not_compactified(100)
assert len(blocks) == 100
stop = time()
total_time += stop - start
print(f"{total_time:0.4f}s, get_random_not_compactified")
all_test_time += total_time
print(f"all tests completed in {all_test_time:0.4f}s")
db_size = os.path.getsize(Path("block-store-benchmark.db"))
print(f"database size: {db_size/1000000:.3f} MB")
finally:
await db_wrapper.db.close()
def create_foliage(
constants: ConsensusConstants,
reward_block_unfinished: RewardChainBlockUnfinished,
block_generator: Optional[BlockGenerator],
aggregate_sig: G2Element,
additions: List[Coin],
removals: List[Coin],
prev_block: Optional[BlockRecord],
blocks: BlockchainInterface,
total_iters_sp: uint128,
timestamp: uint64,
farmer_reward_puzzlehash: bytes32,
pool_target: PoolTarget,
get_plot_signature: Callable[[bytes32, G1Element], G2Element],
get_pool_signature: Callable[[PoolTarget, Optional[G1Element]], Optional[G2Element]],
seed: bytes32 = b"",
) -> Tuple[Foliage, Optional[FoliageTransactionBlock], Optional[TransactionsInfo]]:
"""
Creates a foliage for a given reward chain block. This may or may not be a tx block. In the case of a tx block,
the return values are not None. This is called at the signage point, so some of this information may be
tweaked at the infusion point.
Args:
constants: consensus constants being used for this chain
reward_block_unfinished: the reward block to look at, potentially at the signage point
block_generator: transactions to add to the foliage block, if created
aggregate_sig: aggregate of all transctions (or infinity element)
prev_block: the previous block at the signage point
blocks: dict from header hash to blocks, of all ancestor blocks
total_iters_sp: total iters at the signage point
timestamp: timestamp to put into the foliage block
farmer_reward_puzzlehash: where to pay out farming reward
pool_target: where to pay out pool reward
get_plot_signature: retrieve the signature corresponding to the plot public key
get_pool_signature: retrieve the signature corresponding to the pool public key
seed: seed to randomize block
"""
if prev_block is not None:
res = get_prev_transaction_block(prev_block, blocks, total_iters_sp)
is_transaction_block: bool = res[0]
prev_transaction_block: Optional[BlockRecord] = res[1]
else:
# Genesis is a transaction block
prev_transaction_block = None
is_transaction_block = True
random.seed(seed)
# Use the extension data to create different blocks based on header hash
extension_data: bytes32 = random.randint(0, 100000000).to_bytes(32, "big")
if prev_block is None:
height: uint32 = uint32(0)
else:
height = uint32(prev_block.height + 1)
# Create filter
byte_array_tx: List[bytes32] = []
tx_additions: List[Coin] = []
tx_removals: List[bytes32] = []
pool_target_signature: Optional[G2Element] = get_pool_signature(
pool_target, reward_block_unfinished.proof_of_space.pool_public_key
)
foliage_data = FoliageBlockData(
reward_block_unfinished.get_hash(),
pool_target,
pool_target_signature,
farmer_reward_puzzlehash,
extension_data,
)
foliage_block_data_signature: G2Element = get_plot_signature(
foliage_data.get_hash(),
reward_block_unfinished.proof_of_space.plot_public_key,
)
prev_block_hash: bytes32 = constants.GENESIS_CHALLENGE
if height != 0:
assert prev_block is not None
prev_block_hash = prev_block.header_hash
generator_block_heights_list: List[uint32] = []
if is_transaction_block:
cost = uint64(0)
# Calculate the cost of transactions
if block_generator is not None:
generator_block_heights_list = block_generator.block_height_list()
result: NPCResult = get_name_puzzle_conditions(block_generator, constants.MAX_BLOCK_COST_CLVM, True)
cost = calculate_cost_of_program(block_generator.program, result, constants.COST_PER_BYTE)
removal_amount = 0
addition_amount = 0
for coin in removals:
removal_amount += coin.amount
for coin in additions:
addition_amount += coin.amount
spend_bundle_fees = removal_amount - addition_amount
else:
spend_bundle_fees = 0
reward_claims_incorporated = []
if height > 0:
assert prev_transaction_block is not None
assert prev_block is not None
curr: BlockRecord = prev_block
while not curr.is_transaction_block:
curr = blocks.block_record(curr.prev_hash)
assert curr.fees is not None
pool_coin = create_pool_coin(
curr.height, curr.pool_puzzle_hash, calculate_pool_reward(curr.height), constants.GENESIS_CHALLENGE
)
farmer_coin = create_farmer_coin(
curr.height,
curr.farmer_puzzle_hash,
uint64(calculate_base_farmer_reward(curr.height) + curr.fees),
constants.GENESIS_CHALLENGE,
)
assert curr.header_hash == prev_transaction_block.header_hash
reward_claims_incorporated += [pool_coin, farmer_coin]
if curr.height > 0:
curr = blocks.block_record(curr.prev_hash)
# Prev block is not genesis
while not curr.is_transaction_block:
pool_coin = create_pool_coin(
curr.height,
curr.pool_puzzle_hash,
calculate_pool_reward(curr.height),
constants.GENESIS_CHALLENGE,
)
farmer_coin = create_farmer_coin(
curr.height,
curr.farmer_puzzle_hash,
calculate_base_farmer_reward(curr.height),
constants.GENESIS_CHALLENGE,
)
reward_claims_incorporated += [pool_coin, farmer_coin]
curr = blocks.block_record(curr.prev_hash)
additions.extend(reward_claims_incorporated.copy())
for coin in additions:
tx_additions.append(coin)
byte_array_tx.append(bytearray(coin.puzzle_hash))
for coin in removals:
tx_removals.append(coin.name())
byte_array_tx.append(bytearray(coin.name()))
bip158: PyBIP158 = PyBIP158(byte_array_tx)
encoded = bytes(bip158.GetEncoded())
removal_merkle_set = MerkleSet()
addition_merkle_set = MerkleSet()
# Create removal Merkle set
for coin_name in tx_removals:
removal_merkle_set.add_already_hashed(coin_name)
# Create addition Merkle set
puzzlehash_coin_map: Dict[bytes32, List[Coin]] = {}
for coin in tx_additions:
if coin.puzzle_hash in puzzlehash_coin_map:
puzzlehash_coin_map[coin.puzzle_hash].append(coin)
else:
puzzlehash_coin_map[coin.puzzle_hash] = [coin]
# Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash
for puzzle, coins in puzzlehash_coin_map.items():
addition_merkle_set.add_already_hashed(puzzle)
addition_merkle_set.add_already_hashed(hash_coin_list(coins))
additions_root = addition_merkle_set.get_root()
removals_root = removal_merkle_set.get_root()
generator_hash = bytes32([0] * 32)
if block_generator is not None:
generator_hash = std_hash(block_generator.program)
generator_refs_hash = bytes32([1] * 32)
if generator_block_heights_list not in (None, []):
generator_ref_list_bytes = b"".join([bytes(i) for i in generator_block_heights_list])
generator_refs_hash = std_hash(generator_ref_list_bytes)
filter_hash: bytes32 = std_hash(encoded)
transactions_info: Optional[TransactionsInfo] = TransactionsInfo(
generator_hash,
generator_refs_hash,
aggregate_sig,
uint64(spend_bundle_fees),
cost,
reward_claims_incorporated,
)
if prev_transaction_block is None:
prev_transaction_block_hash: bytes32 = constants.GENESIS_CHALLENGE
else:
prev_transaction_block_hash = prev_transaction_block.header_hash
assert transactions_info is not None
foliage_transaction_block: Optional[FoliageTransactionBlock] = FoliageTransactionBlock(
prev_transaction_block_hash,
timestamp,
filter_hash,
additions_root,
removals_root,
transactions_info.get_hash(),
)
assert foliage_transaction_block is not None
foliage_transaction_block_hash: Optional[bytes32] = foliage_transaction_block.get_hash()
foliage_transaction_block_signature: Optional[G2Element] = get_plot_signature(
foliage_transaction_block_hash, reward_block_unfinished.proof_of_space.plot_public_key
)
assert foliage_transaction_block_signature is not None
else:
foliage_transaction_block_hash = None
foliage_transaction_block_signature = None
foliage_transaction_block = None
transactions_info = None
assert (foliage_transaction_block_hash is None) == (foliage_transaction_block_signature is None)
foliage = Foliage(
prev_block_hash,
reward_block_unfinished.get_hash(),
foliage_data,
foliage_block_data_signature,
foliage_transaction_block_hash,
foliage_transaction_block_signature,
)
return foliage, foliage_transaction_block, transactions_info
),
g2_element,
)
foliage_transaction_block = FoliageTransactionBlock(
bytes32(
bytes.fromhex(
"852ed117f46fa98af7a17fcb050c369245a30fcffc190177c3a316109d1609c7")
),
uint64(3871668531533889186),
bytes32(
bytes.fromhex(
"ffab724c5df9b90c0842565225f5ed842da14f159373c05d63643405ccce84b3")
),
bytes32(
bytes.fromhex(
"5f87a17fafb44afd0d6b5b67b77be38570b4bc0150388bd9c176d4ac5d4e693b")
),
bytes32(
bytes.fromhex(
"db967ce278f9bf4fdc77cb9fa82b5b2ce6876746eb5e61f4352a41e3abb63275")
),
bytes32(
bytes.fromhex(
"7eebe3b21505f7c7cb5536e96ab893bfa4626a5cf9c79fadb5dae6913e0a7cb3")
),
)
end_of_subslot_bundle = EndOfSubSlotBundle(
challenge_chain,
infused_challenge_chain,