def test_ibd_integration(caplog):
# just testing the basics: if we set up 2 threads, one with a coinstate, and one without, will the chain propagate?
caplog.set_level(logging.INFO)
coinstate = _read_chain_from_disk(5)
thread_a = NetworkingThread(coinstate, 12412, FakeDiskInterface())
thread_a.start()
_try_to_connect('127.0.0.1', 12412)
thread_b = NetworkingThread(CoinState.zero(), 12413, FakeDiskInterface())
thread_b.local_peer.network_manager.disconnected_peers = load_peers_from_list(
[('127.0.0.1', 12412, "OUTGOING")])
thread_b.start()
try:
start_time = time()
while True:
if thread_b.local_peer.chain_manager.coinstate.head().height == 5:
break
if time() > start_time + 5:
print("\n".join(str(r) for r in caplog.records))
raise Exception("IBD failed")
sleep(0.01)
finally:
thread_a.stop()
thread_a.join()
thread_b.stop()
thread_b.join()
def read_chain_from_disk() -> CoinState:
if os.path.isfile('chain.cache'):
print("Reading cached chain")
with open('chain.cache', 'rb') as file:
coinstate = CoinState.load(lambda: pickle.load(file))
else:
coinstate = CoinState.zero()
rewrite = False
if os.path.isdir('chain'):
# the code below is no longer needed by normal users, but some old testcases still rely on it:
for filename in sorted(os.listdir('chain')):
(height_str, hash_str) = filename.split("-")
(height, hash) = (int(height_str), computer(hash_str))
if hash not in coinstate.block_by_hash:
if height % 1000 == 0:
print(filename)
if os.path.getsize(f"chain/{filename}") == 0:
print("Stopping at empty block file: %s" % filename)
break
with open(Path("chain") / filename, 'rb') as f:
try:
block = Block.stream_deserialize(f)
except Exception as e:
raise Exception("Corrupted block on disk: %s" % filename) from e
try:
coinstate = coinstate.add_block_no_validation(block)
except Exception:
print("Failed to add block at height=%d, previous_hash=%s"
% (block.height, human(block.header.summary.previous_block_hash)))
break
rewrite = True
if rewrite:
DiskInterface().write_chain_cache_to_disk(coinstate)
return coinstate
def build_pkb2(coinstate: CoinState) -> immutables.Map[PublicKey, PKBalance2]:
public_key_balances_2: immutables.Map[PublicKey,
PKBalance2] = immutables.Map()
for height in range(coinstate.head().height + 1):
block = coinstate.at_head.block_by_height[height]
public_key_balances_2 = build_pkb2_block(coinstate, block,
public_key_balances_2)
return public_key_balances_2
def test_construct_pow_evidence_genesis_block():
coinstate = CoinState.empty()
transactions = [
construct_coinbase_transaction(0, [], immutables.Map(), b"Political statement goes here", example_public_key),
]
summary = construct_minable_summary_genesis(transactions, 1231006505, 0)
evidence = construct_pow_evidence(coinstate, summary, 0, transactions)
evidence.serialize()
def _read_chain_from_disk(max_height):
coinstate = CoinState.zero()
for file_path in sorted(CHAIN_TESTDATA_PATH.iterdir()):
height = int(file_path.name.split("-")[0])
if height > max_height:
return coinstate
block = Block.stream_deserialize(open(file_path, 'rb'))
coinstate = coinstate.add_block_no_validation(block)
return coinstate
def read_chain_from_disk():
print("Reading chain from disk")
coinstate = CoinState.zero()
for filename in sorted(os.listdir('chain')):
height = int(filename.split("-")[0])
if height % 1000 == 0:
print(filename)
block = Block.stream_deserialize(open(Path('chain') / filename, 'rb'))
coinstate = coinstate.add_block_no_validation(block)
return coinstate
def _read_chain_from_disk(max_height):
# the requirement to run these tests from an environment that has access to the real blockchain is hard-coded (for
# now)
coinstate = CoinState.zero()
for filename in sorted(os.listdir('chain')):
height = int(filename.split("-")[0])
if height > max_height:
return coinstate
block = Block.stream_deserialize(open(Path('chain') / filename, 'rb'))
coinstate = coinstate.add_block_no_validation(block)
return coinstate
def read_chain_from_disk():
print("Reading chain from disk")
coinstate = CoinState.zero()
for filename in sorted(os.listdir('chain')):
height = int(filename.split("-")[0])
if height % 1000 == 0:
print(filename)
try:
block = Block.stream_deserialize(open(Path('chain') / filename, 'rb'))
except Exception as e:
raise Exception("Corrupted block on disk: %s" % filename) from e
coinstate = coinstate.add_block_no_validation(block)
return coinstate
def test_validate_block_by_itself_invalid_coinbase_transaction():
coinstate = CoinState.empty()
transactions = [
Transaction(inputs=[
Input(
OutputReference(b'x' * 32, 1),
SECP256k1Signature(b'y' * 64),
)
],
outputs=[Output(1, example_public_key)])
]
summary = construct_minable_summary(coinstate, transactions, 1615209942,
39)
evidence = construct_pow_evidence(coinstate, summary, 0, transactions)
block = Block(BlockHeader(summary, evidence), transactions)
with pytest.raises(ValidateTransactionError, match=".*thin air.*"):
validate_block_by_itself(block, 1615209942)
def write_chain_cache_to_disk(self, coinstate: CoinState) -> None:
# Currently this takes about 2 seconds. It could be optimized further
# if we switch to an appendable file format for the cache.
with open('chain.cache.tmp', 'wb') as file:
coinstate.dump(lambda data: pickle.dump(data, file))
os.replace('chain.cache.tmp', 'chain.cache')
def read_chain_from_disk() -> CoinState:
if os.path.isfile('chain.cache'):
print("Reading cached chain")
with open('chain.cache', 'rb') as file:
coinstate = CoinState.load(lambda: pickle.load(file))
else:
try:
print("Pre-download blockchain from trusted source to 'chain.zip'")
with urllib.request.urlopen(TRUSTED_BLOCKCHAIN_ZIP) as resp:
with open('chain.zip', 'wb') as outfile:
outfile.write(resp.read())
print("Reading initial chain from zipfile")
coinstate = CoinState.zero()
with zipfile.ZipFile('chain.zip') as zip:
for entry in zip.infolist():
if not entry.is_dir():
filename = entry.filename.split('/')[1]
height = int(filename.split("-")[0])
if height % 1000 == 0:
print(filename)
data = zip.read(entry)
block = Block.stream_deserialize(BytesIO(data))
coinstate = coinstate.add_block_no_validation(block)
except Exception:
print(
"Error reading zip file. We'll start with an empty blockchain instead."
+ traceback.format_exc())
coinstate = CoinState.zero()
rewrite = False
if os.path.isdir('chain'):
# the code below is no longer needed by normal users, but some old testcases still rely on it:
for filename in sorted(os.listdir('chain')):
(height_str, hash_str) = filename.split("-")
(height, hash) = (int(height_str), computer(hash_str))
if hash not in coinstate.block_by_hash:
if height % 1000 == 0:
print(filename)
if os.path.getsize(f"chain/{filename}") == 0:
print("Stopping at empty block file: %s" % filename)
break
with open(Path("chain") / filename, 'rb') as f:
try:
block = Block.stream_deserialize(f)
except Exception as e:
raise Exception("Corrupted block on disk: %s" %
filename) from e
try:
coinstate = coinstate.add_block_no_validation(block)
except Exception:
print(
"Failed to add block at height=%d, previous_hash=%s"
%
(block.height,
human(block.header.summary.previous_block_hash)))
break
rewrite = True
if rewrite:
DiskInterface().write_chain_cache_to_disk(coinstate)
return coinstate
def test_faster_read():
with open('test.tmp', 'rb') as file:
with cProfile.Profile() as pr:
pr.runcall(lambda: CoinState.load(lambda: pickle.load(file)))
pr.print_stats()
def build_explorer(coinstate: CoinState) -> None:
explorer_dir = Path(os.environ["EXPLORER_DIR"])
public_key_balances_2: immutables.Map[PublicKey,
PKBalance2] = immutables.Map()
for height in range(coinstate.head().height + 1):
print("Block", height)
block = coinstate.at_head.block_by_height[height]
potential_message = block.transactions[0].inputs[0].signature.signature
if all([(32 <= b < 127) or (b == 10) for b in potential_message]):
msg = "```\n" + str(potential_message, encoding="ascii") + "\n```"
else:
msg = ""
unspent_transaction_outs = get_unspent_transaction_outs_before_block(
coinstate, block)
public_key_balances_2 = build_pkb2_block(coinstate, block,
public_key_balances_2)
with open(explorer_dir / (human(block.hash()) + '.md'),
'w') as block_f:
block_f.write(f"""## Block {human(block.hash())}
Attribute | Value
--- | ---
Height | {block.height}
Hash | {human(block.hash())}
Timestamp | {datetime.fromtimestamp(block.timestamp, tz=timezone.utc).isoformat()}
Target | {human(block.target)}
Merke root | {human(block.merkle_root_hash)}
Nonce | {block.nonce}
{msg}
### Transactions
Hash | Amount
--- | ---
""")
for transaction in block.transactions:
h = human(transaction.hash())
v = show_coin(sum(o.value for o in transaction.outputs))
block_f.write(f"""[{h}]({h}.md) | {v} \n""")
with open(explorer_dir / (human(transaction.hash()) + ".md"),
'w') as transaction_f:
transaction_f.write(
f"""## Transaction {human(transaction.hash())}
In block [{human(block.hash())}]({human(block.hash())}.md)
### Inputs
Transaction | Output Index | Value | Address
--- | --- | --- | ---
""")
for input in transaction.inputs:
output_reference = input.output_reference
if output_reference.hash != 32 * b'\x00':
output: Output = unspent_transaction_outs[
output_reference]
h = human(output_reference.hash)
v = show_coin(output.value)
a = "SKE" + human(
output.public_key.public_key) + "PTI"
transaction_f.write(
f"""[{h}]({h}.md) | {output_reference.index} | """
f"""{v} | [{a}]({a}.md)\n""")
else:
h = human(output_reference.hash)
v = ""
a = "Thin Air"
transaction_f.write(
f"""{h} | {output_reference.index} | """
f"""{v} | {a}\n""")
transaction_f.write("""### Outputs
Value | Address
--- | ---
""")
for output in transaction.outputs:
v = show_coin(output.value)
a = "SKE" + human(output.public_key.public_key) + "PTI"
transaction_f.write(f"""{v} | [{a}]({a}.md)\n""")
for pk, pkb2 in public_key_balances_2.items():
v = show_coin(pkb2.value)
address = "SKE" + human(pk.public_key) + "PTI"
with open(explorer_dir / (address + ".md"), 'w') as address_f:
address_f.write(f"""## {address}
Current balance: {v}
(as of block {coinstate.head().height})
## Received in
Transaction | Output Index
--- | ---
""")
for output_reference in pkb2.all_output_references:
h = human(output_reference.hash)
address_f.write(
f"""[{h}]({h}.md) | {output_reference.index}\n""")
if len(pkb2.spent_in_transactions) > 0:
address_f.write("""
## Spent in
Transaction | ...
--- | ---
""")
for transaction in pkb2.spent_in_transactions:
address_f.write(f"""{human(transaction.hash())} | ...\n""")
else:
address_f.write("""
## Spent in
-- not spent --
""")