def test_journal_db_diff_application_mimics_persist(journal_db, memory_db,
actions):
memory_db.kv_store.clear(
) # hypothesis isn't resetting the other test-scoped fixtures
for action in actions:
if action is DO_RECORD:
journal_db.record()
elif len(action) == 1:
try:
del journal_db[action]
except KeyError:
pass
elif len(action) == 2:
key, val = action
journal_db.set(key, val)
else:
raise Exception("Incorrectly formatted fixture input: %r" % action)
assert MemoryDB({}) == memory_db
diff = journal_db.diff()
journal_db.persist()
diff_test_db = MemoryDB()
diff.apply_to(diff_test_db)
assert memory_db == diff_test_db
def fork_chain(chain):
# make a duplicate chain with no shared state
fork_db = MemoryDB(chain.chaindb.db.kv_store.copy())
fork_chain = type(chain)(fork_db, chain.header)
# sanity check to verify that the two chains are the same.
assert chain.header == fork_chain.header
assert chain.header.block_number == 4
assert fork_chain.header.block_number == 4
assert fork_chain.get_canonical_head() == chain.get_canonical_head()
block_0 = chain.get_canonical_block_by_number(0)
assert fork_chain.get_canonical_block_by_number(0) == block_0
block_1 = chain.get_canonical_block_by_number(1)
assert fork_chain.get_canonical_block_by_number(1) == block_1
block_2 = chain.get_canonical_block_by_number(2)
assert fork_chain.get_canonical_block_by_number(2) == block_2
block_3 = chain.get_canonical_block_by_number(3)
assert fork_chain.get_canonical_block_by_number(3) == block_3
return fork_chain
async def test_syncer_requests_new_collations(request, event_loop):
# setup a-b topology
peer_a_b, peer_b_a = await get_directly_linked_sharding_peers(
request, event_loop)
peer_a_b_subscriber = asyncio.Queue()
peer_a_b.add_subscriber(peer_a_b_subscriber)
peer_pool_b = MockPeerPoolWithConnectedPeers([peer_b_a])
# setup shard dbs at b
shard_db = ShardDB(MemoryDB())
shard = Shard(shard_db, 0)
# start shard syncer
syncer = ShardSyncer(shard, peer_pool_b)
asyncio.ensure_future(syncer.run())
def finalizer():
event_loop.run_until_complete(syncer.cancel())
request.addfinalizer(finalizer)
# notify b about new hashes at a and check that it requests them
hashes_and_periods = ((b"\xaa" * 32, 0), )
peer_a_b.sub_proto.send_new_collation_hashes(hashes_and_periods)
peer, cmd, msg = await asyncio.wait_for(
peer_a_b_subscriber.get(),
timeout=1,
)
assert peer == peer_a_b
assert isinstance(cmd, GetCollations)
assert msg["collation_hashes"] == (hashes_and_periods[0][0], )
async def test_syncer_proposing(request, event_loop):
# setup a-b topology
peer_a_b, peer_b_a = await get_directly_linked_sharding_peers(
request, event_loop)
peer_a_b_subscriber = asyncio.Queue()
peer_a_b.add_subscriber(peer_a_b_subscriber)
peer_pool_b = MockPeerPoolWithConnectedPeers([peer_b_a])
# setup shard dbs at b
shard_db = ShardDB(MemoryDB())
shard = Shard(shard_db, 0)
# start shard syncer
syncer = ShardSyncer(shard, peer_pool_b)
asyncio.ensure_future(syncer.run())
def finalizer():
event_loop.run_until_complete(syncer.cancel())
request.addfinalizer(finalizer)
# propose at b and check that it announces its proposal
await syncer.propose()
peer, cmd, msg = await asyncio.wait_for(
peer_a_b_subscriber.get(),
timeout=1,
)
assert peer == peer_a_b
assert isinstance(cmd, NewCollationHashes)
assert len(msg["collation_hashes_and_periods"]) == 1
proposed_hash = msg["collation_hashes_and_periods"][0][0]
# test that the collation has been added to the shard
shard.get_collation_by_hash(proposed_hash)
def test_journal_db_rejects_committing_root():
memory_db = MemoryDB({})
journal_db = JournalDB(memory_db)
root = journal_db.journal.root_changeset_id
with pytest.raises(ValidationError):
journal_db.commit(root)
async def test_peer_pool_connect(monkeypatch, event_loop,
receiver_server_with_dumb_peer):
started_peers = []
async def mock_start_peer(peer):
nonlocal started_peers
started_peers.append(peer)
monkeypatch.setattr(receiver_server_with_dumb_peer, '_start_peer',
mock_start_peer)
# We need this to ensure the server can check if the peer pool is full for
# incoming connections.
monkeypatch.setattr(receiver_server_with_dumb_peer, 'peer_pool',
MockPeerPool())
pool = PeerPool(DumbPeer, FakeAsyncHeaderDB(MemoryDB()), NETWORK_ID,
INITIATOR_PRIVKEY, tuple())
nodes = [RECEIVER_REMOTE]
await pool.connect_to_nodes(nodes)
# Give the receiver_server a chance to ack the handshake.
await asyncio.sleep(0.1)
assert len(started_peers) == 1
assert len(pool.connected_nodes) == 1
# Stop our peer to make sure its pending asyncio tasks are cancelled.
await list(pool.connected_nodes.values())[0].cancel()
def prepare_computation(vm_class):
message = Message(
to=CANONICAL_ADDRESS_A,
sender=CANONICAL_ADDRESS_B,
value=100,
data=b'',
code=b'',
gas=800,
)
tx_context = vm_class._state_class.transaction_context_class(
gas_price=1,
origin=CANONICAL_ADDRESS_B,
)
vm = vm_class(GENESIS_HEADER, ChainDB(MemoryDB()))
computation = vm_class._state_class.computation_class(
state=vm.state,
message=message,
transaction_context=tx_context,
)
computation.state.account_db.touch_account(
decode_hex(EMPTY_ADDRESS_IN_STATE))
computation.state.account_db.set_code(decode_hex(ADDRESS_WITH_CODE[0]),
ADDRESS_WITH_CODE[1])
return computation
async def test_witness_history_on_repeat_blocks():
"""
Repeated blocks should not consume more slots in the limited history of block witnesses
"""
wit_db = AsyncWitnessDB(MemoryDB())
hash1 = Hash32Factory()
hash1_witnesses = tuple(Hash32Factory.create_batch(5))
await wit_db.coro_persist_witness_hashes(hash1, hash1_witnesses)
hash2 = Hash32Factory()
await wit_db.coro_persist_witness_hashes(hash2, tuple(Hash32Factory.create_batch(5)))
# *almost* push the first witness out of history
for _ in range(wit_db._max_witness_history - 2):
await wit_db.coro_persist_witness_hashes(Hash32Factory(), Hash32Factory.create_batch(2))
# It should still be there...
loaded_hashes = await wit_db.coro_get_witness_hashes(hash1)
assert set(loaded_hashes) == set(hash1_witnesses)
# Add one more new witness, for an existing block
await wit_db.coro_persist_witness_hashes(hash2, Hash32Factory.create_batch(2))
# That new witness should *not* consume a block slot in history, so the first hash's
# witness should still be available.
loaded_hashes = await wit_db.coro_get_witness_hashes(hash1)
assert set(loaded_hashes) == set(hash1_witnesses)
def __init__(self, bootnodes):
privkey = eth_keys.keys.PrivateKey(keccak(b"seed"))
self.messages = []
enr_db = ENRDB(MemoryDB(), default_identity_scheme_registry)
socket = trio.socket.socket(family=trio.socket.AF_INET, type=trio.socket.SOCK_DGRAM)
event_bus = None
address = AddressFactory()
super().__init__(
privkey, address.udp_port, address.tcp_port, bootnodes, event_bus, socket, enr_db)
def test_journal_db_diff_respects_clear():
memory_db = MemoryDB({})
journal_db = JournalDB(memory_db)
journal_db[b'first'] = b'val'
journal_db.clear()
pending = journal_db.diff().pending_items()
assert len(pending) == 0
async def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('-bootnode',
type=str,
help="The enode to use as bootnode")
parser.add_argument('-networkid',
type=int,
choices=[ROPSTEN_NETWORK_ID, MAINNET_NETWORK_ID],
default=ROPSTEN_NETWORK_ID,
help="1 for mainnet, 3 for testnet")
parser.add_argument('-l', type=str, help="Log level", default="info")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s: %(message)s',
datefmt='%H:%M:%S')
if args.l == "debug2": # noqa: E741
log_level = DEBUG2_LEVEL_NUM
else:
log_level = getattr(logging, args.l.upper())
logging.getLogger('p2p').setLevel(log_level)
network_cfg = PRECONFIGURED_NETWORKS[args.networkid]
# Listen on a port other than 30303 so that we can test against a local geth instance
# running on that port.
listen_port = 30304
# Use a hard-coded privkey so that our enode is always the same.
privkey = keys.PrivateKey(
b'~\x054{4\r\xd64\x0f\x98\x1e\x85;\xcc\x08\x1eQ\x10t\x16\xc0\xb0\x7f)=\xc4\x1b\xb7/\x8b&\x83'
) # noqa: E501
addr = kademlia.Address('127.0.0.1', listen_port, listen_port)
if args.bootnode:
bootstrap_nodes = tuple([kademlia.Node.from_uri(args.bootnode)])
else:
bootstrap_nodes = tuple(
kademlia.Node.from_uri(enode) for enode in network_cfg.bootnodes)
ipc_path = Path(f"networking-{uuid.uuid4()}.ipc")
networking_connection_config = ConnectionConfig(
name=NETWORKING_EVENTBUS_ENDPOINT, path=ipc_path)
headerdb = TrioHeaderDB(AtomicDB(MemoryDB()))
headerdb.persist_header(network_cfg.genesis_header)
vm_config = network_cfg.vm_configuration
enr_field_providers = (functools.partial(generate_eth_cap_enr_field,
vm_config, headerdb), )
socket = trio.socket.socket(family=trio.socket.AF_INET,
type=trio.socket.SOCK_DGRAM)
await socket.bind(('0.0.0.0', listen_port))
async with TrioEndpoint.serve(networking_connection_config) as endpoint:
service = DiscoveryService(privkey, addr, bootstrap_nodes, endpoint,
socket, enr_field_providers)
service.logger.info("Enode: %s", service.this_node.uri())
async with background_trio_service(service):
await service.manager.wait_finished()
def db(request):
base_db = MemoryDB()
if request.param is JournalDB:
return JournalDB(base_db)
elif request.param is BatchDB:
return BatchDB(base_db)
elif request.param is MemoryDB:
return base_db
else:
raise Exception("Invariant")
async def test_witness_eviction_on_repeat_blocks():
"""
After witnesses are persisted twice for the same block, make sure that eviction
does not cause any failures.
"""
wit_db = AsyncWitnessDB(MemoryDB())
hash_ = Hash32Factory()
await wit_db.coro_persist_witness_hashes(hash_, Hash32Factory.create_batch(2))
await wit_db.coro_persist_witness_hashes(hash_, Hash32Factory.create_batch(2))
for _ in range(wit_db._max_witness_history):
await wit_db.coro_persist_witness_hashes(Hash32Factory(), Hash32Factory.create_batch(2))
def get_chain(vm: Type[BaseVM]) -> MiningChain:
return chain_without_pow(
MemoryDB(), vm, GENESIS_PARAMS,
genesis_state([
AddressSetup(address=FUNDED_ADDRESS,
balance=DEFAULT_INITIAL_BALANCE,
code=b''),
AddressSetup(address=SECOND_EXISTING_ADDRESS,
balance=DEFAULT_INITIAL_BALANCE,
code=b'')
]))
async def test_persisting_and_looking_up():
wit_db = AsyncWitnessDB(MemoryDB())
hash1 = Hash32Factory()
with pytest.raises(WitnessHashesUnavailable):
await wit_db.coro_get_witness_hashes(hash1)
hash1_witnesses = tuple(Hash32Factory.create_batch(5))
await wit_db.coro_persist_witness_hashes(hash1, hash1_witnesses)
loaded_hashes = await wit_db.coro_get_witness_hashes(hash1)
assert set(loaded_hashes) == set(hash1_witnesses)
def main():
args = parse_args()
# print('Called with args:')
# print(args)
# genesis address
init_address = to_canonical_address(
"8888f1f195afa192cfee860698584c030f4c9db1")
base_state = base_genesis_state(init_address,
funded_address_initial_balance())
# just an address
simple_contract_address = create_simple_contract_address()
# create chain
klass = MiningChain.configure(
__name__='MyTestChain',
vm_configuration=((constants.GENESIS_BLOCK_NUMBER,
SpuriousDragonVM), ),
network_id=1337,
)
chain = klass.from_genesis(
MemoryDB(), GENESIS_PARAMS,
genesis_state(base_state, simple_contract_address, args.data))
# TODO
# signature = 'getMeaningOfLife()' # function name
# function_selector = function_signature_to_4byte_selector(signature)
'''
new_transaction(
vm,
from_,
to,
amount=0,
private_key=None,
gas_price=10,
gas=100000,
data=b''
)
'''
call_txn = new_transaction(
chain.get_vm(),
SENDER,
simple_contract_address,
gas_price=0,
# data=function_selector,
data=decode_hex(args.signature),
)
result_bytes = chain.get_transaction_result(call_txn,
chain.get_canonical_head())
def test_chain_config_from_preconfigured_network(network_id):
chain_config = Eth1ChainConfig.from_preconfigured_network(network_id)
chain = chain_config.initialize_chain(AtomicDB(MemoryDB()))
if network_id == MAINNET_NETWORK_ID:
assert chain_config.chain_id == MainnetChain.chain_id
assert_vm_configuration_equal(chain_config.vm_configuration, MainnetChain.vm_configuration)
assert chain.get_canonical_head() == MAINNET_GENESIS_HEADER
elif network_id == ROPSTEN_NETWORK_ID:
assert chain_config.chain_id == RopstenChain.chain_id
assert_vm_configuration_equal(chain_config.vm_configuration, RopstenChain.vm_configuration)
assert chain.get_canonical_head() == ROPSTEN_GENESIS_HEADER
else:
raise AssertionError("Invariant: unreachable code path")
def copy(chain):
"""
Make a copy of the chain at the given state. Actions performed on the
resulting chain will not affect the original chain.
"""
if not isinstance(chain, MiningChain):
raise ValidationError("`at_block_number` may only be used with 'MiningChain")
base_db = chain.chaindb.db
if not isinstance(base_db, MemoryDB):
raise ValidationError("Unsupported database type: {0}".format(type(base_db)))
db = MemoryDB(base_db.kv_store.copy())
chain_copy = type(chain)(db, chain.header)
return chain_copy
def new_chain_from_fixture(fixture):
base_db = MemoryDB()
vm_config = chain_vm_configuration(fixture)
ChainFromFixture = MainnetChain.configure(
'ChainFromFixture',
vm_configuration=vm_config,
)
return ChainFromFixture.from_genesis(
base_db,
genesis_params=genesis_params_from_fixture(fixture),
genesis_state=fixture['pre'],
)
def main ():
args = parse_args()
# print('Called with args:')
# print(args)
# create an vm
base_db = MemoryDB()
# genesis address
init_address = to_canonical_address("8888f1f195afa192cfee860698584c030f4c9db1")
genesis_state = base_genesis_state(init_address, funded_address_initial_balance())
chain = chain_with_block_validation(base_db, genesis_state)
vm = chain.get_vm()
# print('Excuting...')
args_to_bytecode_computation(args, args.code, vm)
def chaindb_20():
chain = PoWMiningChain.from_genesis(MemoryDB(), GENESIS_PARAMS,
GENESIS_STATE)
for i in range(20):
tx = chain.create_unsigned_transaction(
nonce=i,
gas_price=1234,
gas=1234000,
to=RECEIVER.public_key.to_canonical_address(),
value=i,
data=b'',
)
chain.apply_transaction(tx.as_signed_transaction(SENDER))
chain.mine_block()
return chain.chaindb
async def _test_trie_sync():
src_trie, contents = make_random_trie(random)
dest_db = FakeAsyncMemoryDB()
nodes_cache = MemoryDB()
scheduler = HexaryTrieSync(src_trie.root_hash, dest_db, nodes_cache, TraceLogger("test"))
requests = scheduler.next_batch()
while len(requests) > 0:
results = []
for request in requests:
results.append([request.node_key, src_trie.db[request.node_key]])
await scheduler.process(results)
requests = scheduler.next_batch(10)
dest_trie = HexaryTrie(dest_db, src_trie.root_hash)
for key, value in contents.items():
assert dest_trie[key] == value
def base_chain(request, genesis_state):
VMClass = request.param.configure(validate_seal=lambda block: None)
class ChainForTest(MiningChain):
vm_configuration = ((0, VMClass), )
network_id = 1337
genesis_params = {
'block_number': constants.GENESIS_BLOCK_NUMBER,
'difficulty': constants.GENESIS_DIFFICULTY,
'gas_limit': constants.GENESIS_GAS_LIMIT,
}
chain = ChainForTest.from_genesis(MemoryDB(), genesis_params,
genesis_state)
return chain
async def test_witness_union():
wit_db = AsyncWitnessDB(MemoryDB())
hash1 = Hash32Factory()
hash1_witnesses_unique1 = set(Hash32Factory.create_batch(3))
hash1_witnesses_unique2 = set(Hash32Factory.create_batch(3))
hash1_witnesses_both = set(Hash32Factory.create_batch(2))
hash1_witnesses1 = tuple(hash1_witnesses_unique1 | hash1_witnesses_both)
hash1_witnesses2 = tuple(hash1_witnesses_unique2 | hash1_witnesses_both)
await wit_db.coro_persist_witness_hashes(hash1, hash1_witnesses1)
await wit_db.coro_persist_witness_hashes(hash1, hash1_witnesses2)
stored_hashes = await wit_db.coro_get_witness_hashes(hash1)
expected = hash1_witnesses_unique1 | hash1_witnesses_both | hash1_witnesses_unique2
assert set(stored_hashes) == expected
async def _manually_driven_discovery(seed, socket, nursery):
_, port = socket.getsockname()
discovery = ManuallyDrivenDiscoveryService(
keys.PrivateKey(keccak(seed)),
port,
port,
bootstrap_nodes=[],
event_bus=None,
socket=socket,
node_db=NodeDB(default_identity_scheme_registry, MemoryDB()))
async with background_trio_service(discovery):
# Wait until we're fully initialized (i.e. until the ENR stub created in the constructor
# is replaced with the real one).
while discovery.this_node.enr.sequence_number == 0:
await trio.hazmat.checkpoint()
yield discovery
def db(request):
base_db = MemoryDB()
if request.param is JournalDB:
yield JournalDB(base_db)
elif request.param is BatchDB:
yield BatchDB(base_db)
elif request.param is MemoryDB:
yield base_db
elif request.param is AtomicDB:
atomic_db = AtomicDB(base_db)
with atomic_db.atomic_batch() as batch:
yield batch
elif request.param is CacheDB:
yield CacheDB(base_db)
else:
raise Exception("Invariant")
def __init__(self, db: AtomicDatabaseAPI, storage_root: Hash32,
address: Address) -> None:
"""
Database entries go through several pipes, like so...
.. code::
db -> _storage_lookup -> _storage_cache -> _locked_changes -> _journal_storage
db is the raw database, we can assume it hits disk when written to.
Keys are stored as node hashes and rlp-encoded node values.
_storage_lookup is itself a pair of databases: (BatchDB -> HexaryTrie),
writes to storage lookup *are* immeditaely applied to a trie, generating
the appropriate trie nodes and and root hash (via the HexaryTrie). The
writes are *not* persisted to db, until _storage_lookup is explicitly instructed to,
via :meth:`StorageLookup.commit_to`
_storage_cache is a cache tied to the state root of the trie. It
is important that this cache is checked *after* looking for
the key in _journal_storage, because the cache is only invalidated
after a state root change. Otherwise, you will see data since the last
storage root was calculated.
_locked_changes is a batch database that includes only those values that are
un-revertable in the EVM. Currently, that means changes that completed in a
previous transaction.
Journaling batches writes at the _journal_storage layer, until persist is called.
It manages all the checkpointing and rollbacks that happen during EVM execution.
In both _storage_cache and _journal_storage, Keys are set/retrieved as the
big_endian encoding of the slot integer, and the rlp-encoded value.
"""
self._address = address
self._storage_lookup = StorageLookup(db, storage_root, address)
self._storage_cache = CacheDB(self._storage_lookup)
self._locked_changes = JournalDB(self._storage_cache)
self._journal_storage = JournalDB(self._locked_changes)
self._accessed_slots: Set[int] = set()
# Track how many times we have cleared the storage. This is journaled
# in lockstep with other storage changes. That way, we can detect if a revert
# causes use to revert past the previous storage deletion. The clear count is used
# as an index to find the base trie from before the revert.
self._clear_count = JournalDB(
MemoryDB({CLEAR_COUNT_KEY_NAME: to_bytes(0)}))
def test_pow_mining():
sender = keys.PrivateKey(
decode_hex(
"49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
)
receiver = keys.PrivateKey(
decode_hex(
"b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
)
genesis_params = {
'parent_hash': constants.GENESIS_PARENT_HASH,
'uncles_hash': constants.EMPTY_UNCLE_HASH,
'coinbase': constants.ZERO_ADDRESS,
'transaction_root': constants.BLANK_ROOT_HASH,
'receipt_root': constants.BLANK_ROOT_HASH,
'bloom': 0,
'difficulty': 5,
'block_number': constants.GENESIS_BLOCK_NUMBER,
'gas_limit': constants.GENESIS_GAS_LIMIT,
'gas_used': 0,
'timestamp': 1514764800,
'extra_data': constants.GENESIS_EXTRA_DATA,
'nonce': constants.GENESIS_NONCE
}
state = {
sender.public_key.to_canonical_address(): {
"balance": 100000000000000000,
"code": b"",
"nonce": 0,
"storage": {}
}
}
chain = PowMiningChain.from_genesis(MemoryDB(), genesis_params, state)
# import ipdb; ipdb.set_trace()
for i in range(10):
tx = chain.create_unsigned_transaction(
nonce=i,
gas_price=1234,
gas=1234000,
to=receiver.public_key.to_canonical_address(),
value=i,
data=b'',
)
chain.apply_transaction(tx.as_signed_transaction(sender))
block = chain.mine_block()
assert block.number == i + 1
assert chain.header.block_number == i + 2
def get_server(privkey, address, peer_class):
base_db = MemoryDB()
headerdb = FakeAsyncHeaderDB(base_db)
chaindb = ChainDB(base_db)
chaindb.persist_header(ROPSTEN_GENESIS_HEADER)
chain = RopstenChain(base_db)
server = Server(
privkey,
address.tcp_port,
chain,
chaindb,
headerdb,
base_db,
network_id=NETWORK_ID,
peer_class=peer_class,
)
return server
def make_random_state(n):
raw_db = MemoryDB()
account_db = AccountDB(raw_db)
contents = {}
for _ in range(n):
addr = os.urandom(20)
account_db.touch_account(addr)
balance = random.randint(0, 10000)
account_db.set_balance(addr, balance)
nonce = random.randint(0, 10000)
account_db.set_nonce(addr, nonce)
storage = random.randint(0, 10000)
account_db.set_storage(addr, 0, storage)
code = b'not-real-code'
account_db.set_code(addr, code)
contents[addr] = (balance, nonce, storage, code)
account_db.persist()
return raw_db, account_db.state_root, contents
