def test_two_trees():
db = RefcountDB(EphemDB())
NODES = 60
t1 = pruning_trie.Trie(db)
t2 = pruning_trie.Trie(db)
db.ttl = 0
for i in range(NODES):
t1.update(to_string(i), to_string(i))
if i < NODES // 2:
t2.update(to_string(i), to_string(i))
db.commit_refcount_changes(i)
db.cleanup(i)
check_db_tightness([t1, t2], db)
for i in range(NODES):
sys.stderr.write('clearing: %d\n' % i)
t1.delete(to_string(NODES - 1 - i))
db.commit_refcount_changes(NODES + i)
db.cleanup(NODES + i)
check_db_tightness([t1, t2], db)
assert t2.to_dict() == {to_string(i): to_string(i) for i in range(NODES // 2)}
for i in range(NODES // 2):
t2.delete(to_string(i))
db.commit_refcount_changes(NODES * 2 + i)
db.cleanup(NODES * 2 + i)
check_db_tightness([t1, t2], db)
assert len(db.kv) == 0
def __init__(self, app):
self.config = app.config
sce = self.config['eth']
if int(sce['pruning']) >= 0:
self.db = RefcountDB(app.services.db)
if "I am not pruning" in self.db.db:
raise RuntimeError(
"The database in '{}' was initialized as non-pruning. "
"Can not enable pruning now.".format(
self.config['data_dir']))
self.db.ttl = int(sce['pruning'])
self.db.db.put("I am pruning", "1")
else:
self.db = app.services.db
if "I am pruning" in self.db:
raise RuntimeError(
"The database in '{}' was initialized as pruning. "
"Can not disable pruning now".format(
self.config['data_dir']))
self.db.put("I am not pruning", "1")
if 'network_id' in self.db:
db_network_id = self.db.get('network_id')
if db_network_id != str(sce['network_id']):
raise RuntimeError(
"The database in '{}' was initialized with network id {} and can not be used "
"when connecting to network id {}. Please choose a different data directory."
.format(self.config['data_dir'], db_network_id,
sce['network_id']))
else:
self.db.put('network_id', str(sce['network_id']))
self.db.commit()
assert self.db is not None
super(ChainService, self).__init__(app)
log.info('initializing chain')
coinbase = app.services.accounts.coinbase
env = Env(self.db, sce['block'])
self.chain = Chain(env,
new_head_cb=self._on_new_head,
coinbase=coinbase)
log.info('chain at', number=self.chain.head.number)
if 'genesis_hash' in sce:
assert sce['genesis_hash'] == self.chain.genesis.hex_hash(), \
"Genesis hash mismatch.\n Expected: %s\n Got: %s" % (
sce['genesis_hash'], self.chain.genesis.hex_hash())
self.synchronizer = Synchronizer(self, force_sync=None)
self.block_queue = Queue(maxsize=self.block_queue_size)
self.transaction_queue = Queue(maxsize=self.transaction_queue_size)
self.add_blocks_lock = False
self.add_transaction_lock = gevent.lock.Semaphore()
self.broadcast_filter = DuplicatesFilter()
self.on_new_head_cbs = []
self.on_new_head_candidate_cbs = []
self.newblock_processing_times = deque(maxlen=1000)
def test_basic_pruning():
db = RefcountDB(EphemDB())
NODES = 60
t = pruning_trie.Trie(db)
db.ttl = 0
db.logging = True
for i in range(NODES):
t.update(to_string(i), to_string(i))
db.commit_refcount_changes(0)
db.cleanup(0)
check_db_tightness([t], db)
for i in range(NODES):
t.update(to_string(i), to_string(i ** 3))
db.commit_refcount_changes(0)
db.cleanup(0)
check_db_tightness([t], db)
for i in range(NODES):
t.delete(to_string(i))
db.commit_refcount_changes(0)
db.cleanup(0)
check_db_tightness([t], db)
assert len(t.to_dict()) == 0
assert len(db.kv) == 0
def run_test(name):
pairs = load_tests()[name]
def _dec(x):
if utils.is_string(x) and x.startswith(b'0x'):
return utils.decode_hex(x[2:])
return x
pairs['in'] = [(_dec(k), _dec(v)) for k, v in pairs['in']]
deletes = [(k, v) for k, v in pairs['in'] if v is None]
N_PERMUTATIONS = 100
for i, permut in enumerate(itertools.permutations(pairs['in'])):
if i > N_PERMUTATIONS:
break
db = RefcountDB(EphemDB())
db.ttl = 0
t = pruning_trie.Trie(db)
for k, v in permut:
# logger.debug('updating with (%s, %s)' %(k, v))
if v is not None:
t.update(k, v)
else:
t.delete(k)
db.commit_refcount_changes(0)
db.cleanup(0)
# make sure we have deletes at the end
for k, v in deletes:
t.delete(k)
t.clear_all()
db.commit_refcount_changes(1)
db.cleanup(1)
assert len(db.kv) == 0
assert pairs['root'] == b'0x' + utils.encode_hex(t.root_hash), (i, list(permut) + deletes)
def test_deep_inner_branch_deletion():
db = RefcountDB(EphemDB())
db.logging = True
db.ttl = 1
t1 = pruning_trie.Trie(db)
t1.update(b'etherdogecoin', b'\x33' * 50)
t1.update(b'etherdogelot', b'\x44' * 50)
t1.delete(b'etherhouse')
t1.delete(b'etherhouse')
t1.delete(b'etherhouse')
t1.delete(b'etherhouse')
def __init__(self, app):
self.config = app.config
sce = self.config['eth']
if int(sce['pruning']) >= 0:
self.db = RefcountDB(app.services.db)
if "I am not pruning" in self.db.db:
raise Exception("This database was initialized as non-pruning."
" Kinda hard to start pruning now.")
self.db.ttl = int(sce['pruning'])
self.db.db.put("I am pruning", "1")
else:
self.db = app.services.db
if "I am pruning" in self.db:
raise Exception("This database was initialized as pruning."
" Kinda hard to stop pruning now.")
self.db.put("I am not pruning", "1")
if 'network_id' in self.db:
db_network_id = self.db.get('network_id')
if db_network_id != str(sce['network_id']):
raise Exception(
"This database was initialized with network_id {} "
"and can not be used when connecting to network_id {}".
format(db_network_id, sce['network_id']))
else:
self.db.put('network_id', str(sce['network_id']))
self.db.commit()
assert self.db is not None
super(ChainService, self).__init__(app)
log.info('initializing chain')
coinbase = app.services.accounts.coinbase
env = Env(self.db, sce['block'])
self.chain = Chain(env,
new_head_cb=self._on_new_head,
coinbase=coinbase)
log.info('chain at', number=self.chain.head.number)
if 'genesis_hash' in sce:
assert sce['genesis_hash'] == self.chain.genesis.hex_hash()
self.synchronizer = Synchronizer(self, force_sync=None)
self.block_queue = Queue(maxsize=self.block_queue_size)
self.transaction_queue = Queue(maxsize=self.transaction_queue_size)
self.add_blocks_lock = False
self.add_transaction_lock = gevent.lock.Semaphore()
self.broadcast_filter = DuplicatesFilter()
self.on_new_head_cbs = []
self.on_new_head_candidate_cbs = []
self.newblock_processing_times = deque(maxlen=1000)
def test_clear():
db = RefcountDB(EphemDB())
NODES = 60
t = pruning_trie.Trie(db)
db.ttl = 0
for i in range(NODES):
t.update(to_string(i), to_string(i))
db.commit_refcount_changes(i)
db.cleanup(i)
t.clear_all()
db.commit_refcount_changes(NODES)
db.cleanup(NODES)
assert len(db.kv) == 0
def test_two_tries_with_small_root_node():
db = RefcountDB(EphemDB())
db.logging = True
db.ttl = 1
t1 = pruning_trie.Trie(db)
t2 = pruning_trie.Trie(db)
t1.update(b'3', b'5')
t2.update(b'3', b'5')
t1.delete(b'3')
db.commit_refcount_changes(0)
db.cleanup(0)
db.cleanup(1)
db.cleanup(2)
print(db.kv)
print(t2.to_dict())
def test_insert_delete():
for a in (5, 15, 60):
db = RefcountDB(EphemDB())
NODES = a
t1 = pruning_trie.Trie(db)
db.ttl = 0
db.logging = True
for i in range(NODES):
t1.update(to_string(i), to_string(i))
db.commit_refcount_changes(i)
db.cleanup(i)
check_db_tightness([t1], db)
for i in range(NODES):
t1.delete(to_string(NODES - 1 - i))
db.commit_refcount_changes(NODES + i)
db.cleanup(NODES + i)
check_db_tightness([t1], db)
assert len(db.kv) == 0
def test_trie_transfer():
db = RefcountDB(EphemDB())
NODES = 60
t1 = pruning_trie.Trie(db)
db.ttl = NODES * 2
for i in range(NODES):
t1.update(to_string(i), to_string(i))
db.commit_refcount_changes(i)
db.cleanup(i)
t2 = pruning_trie.Trie(db)
t2.root_hash = t1.root_hash
assert t2.to_dict() == {to_string(i): to_string(i) for i in range(NODES)}
for i in range(NODES):
t2.delete(to_string(i))
db.commit_refcount_changes(NODES + i)
db.cleanup(NODES + i)
for i in range(NODES * 2):
db.cleanup(2 * NODES + i)
assert len(db.kv) == 0
def __init__(self, app):
self.config = app.config
sce = self.config['eth']
if int(sce['pruning']) >= 0:
self.db = RefcountDB(app.services.db)
if "I am not pruning" in self.db.db:
raise Exception("This database was initialized as non-pruning."
" Kinda hard to start pruning now.")
self.db.ttl = int(sce['pruning'])
self.db.db.put("I am pruning", "1")
else:
self.db = app.services.db
if "I am pruning" in self.db:
raise Exception("This database was initialized as pruning."
" Kinda hard to stop pruning now.")
self.db.put("I am not pruning", "1")
assert self.db is not None
super(ChainService, self).__init__(app)
log.info('initializing chain')
coinbase = app.services.accounts.coinbase
if sce['genesis']:
log.info('loading genesis', path=sce['genesis'])
_json = json.load(open(sce['genesis']))
else:
log.info('loaded default genesis alloc')
_json = None
_genesis = genesis(self.db, json=_json)
log.info('created genesis block', hash=encode_hex(_genesis.hash))
self.chain = Chain(self.db,
genesis=_genesis,
new_head_cb=self._on_new_head,
coinbase=coinbase)
log.info('chain at', number=self.chain.head.number)
self.synchronizer = Synchronizer(self, force_sync=None)
self.block_queue = Queue(maxsize=self.block_queue_size)
self.transaction_queue = Queue(maxsize=self.transaction_queue_size)
self.add_blocks_lock = False
self.add_transaction_lock = gevent.lock.Semaphore()
self.broadcast_filter = DuplicatesFilter()
self.on_new_head_cbs = []
self.on_new_head_candidate_cbs = []
self.newblock_processing_times = deque(maxlen=1000)
def test_block_18315_changes():
pre = {}
toadd = [
['0x0000000000000000000000000000000000000000000000000000000000000000', '0xf9e88bc2b3203e764fe67b4d0f4171b7756117c8'],
['0x0000000000000000000000000000000000000000000000000000000000000001', '0x'],
['0x0000000000000000000000000000000000000000000000000000000000000002', '0x'],
]
db = RefcountDB(EphemDB())
db.logging = True
NODES = 60
t1 = pruning_trie.Trie(db)
t2 = pruning_trie.Trie(db)
db.ttl = NODES * 2
c = 0
for k, v in pre.items():
triekey = utils.sha3(utils.zpad(k[2:].decode('hex'), 32))
t1.update(triekey, rlp.encode(v[2:].decode('hex')))
t2.update(triekey, rlp.encode(v[2:].decode('hex')))
db.commit_refcount_changes(c)
db.cleanup(c)
c += 1
sys.stderr.write('##############################\n')
print(utils.encode_hex(t1.root_hash))
print(t1.to_dict())
for k, v in toadd:
sys.stderr.write('kv: %s %s\n' % (k, v))
triekey = utils.sha3(utils.zpad(utils.decode_hex(k[2:]), 32))
if v == '0x':
t1.delete(triekey)
else:
t1.update(triekey, rlp.encode(utils.decode_hex(v[2:])))
db.commit_refcount_changes(c)
db.cleanup(c)
c += 1
t1.clear_all()
db.commit_refcount_changes(c)
for i in range(db.ttl + 1):
db.cleanup(c)
c += 1
t3 = pruning_trie.Trie(db)
t3.root_hash = t2.root_hash
print(t3.to_dict())
def test_shared_prefix():
db = RefcountDB(EphemDB())
db.logging = True
db.ttl = 1
t1 = pruning_trie.Trie(db)
t2 = pruning_trie.Trie(db)
t1.update(b'dogecoin', b'\x33' * 50)
t1.update(b'dogelot', b'\x44' * 50)
t2.update(b'dogecoin', b'\x33' * 50)
t2.update(b'dogelot', b'\x44' * 50)
print(db.kv)
t1.delete(b'dogecoin')
t1.delete(b'dogelot')
print(db.kv)
db.commit_refcount_changes(0)
db.cleanup(0)
db.cleanup(1)
db.cleanup(2)
print(db.kv)
print(t2.to_dict())
def test_revert_deletes():
db = RefcountDB(EphemDB())
NODES = 60
t1 = pruning_trie.Trie(db)
db.ttl = NODES * 2
for i in range(NODES):
t1.update(to_string(i), to_string(i))
db.commit_refcount_changes(i)
db.cleanup(i)
x = t1.root_hash
for i in range(NODES):
t1.delete(to_string(i))
db.commit_refcount_changes(NODES + i)
db.cleanup(NODES + i)
for i in range(NODES * 2 - 1, NODES - 1, -1):
db.revert_refcount_changes(i)
for i in range(NODES * 2):
db.cleanup(NODES + i)
db.revert_refcount_changes(i)
t1.root_hash = x
assert t1.to_dict() == {to_string(i): to_string(i) for i in range(NODES)}
def test_delayed_pruning():
NODES = 60
db = RefcountDB(EphemDB())
t = pruning_trie.Trie(db)
db.ttl = NODES // 4
for i in range(NODES):
t.update(to_string(i), to_string(i))
db.commit_refcount_changes(i)
db.cleanup(i)
for i in range(NODES):
t.update(to_string(i), to_string(i ** 3))
db.commit_refcount_changes(i + NODES)
db.cleanup(i + NODES)
for i in range(NODES):
t.delete(to_string(i))
db.commit_refcount_changes(i + NODES * 2)
db.cleanup(i + NODES * 2)
for i in range(NODES // 4):
db.cleanup(i + NODES * 3)
assert len(t.to_dict()) == 0
assert len(db.kv) == 0
def test_two_trees_with_clear():
db = RefcountDB(EphemDB())
NODES = 60
t1 = pruning_trie.Trie(db)
t2 = pruning_trie.Trie(db)
db.ttl = NODES // 4
for i in range(NODES):
t1.update(to_string(i), to_string(i))
if i < NODES // 2:
t2.update(to_string(i), to_string(i))
db.commit_refcount_changes(i)
db.cleanup(i)
t1.clear_all()
db.cleanup(NODES)
assert t2.to_dict() == {to_string(i): to_string(i) for i in range(NODES // 2)}
for i in range(NODES // 2):
t2.delete(to_string(i))
db.commit_refcount_changes(NODES + i)
db.cleanup(NODES + i)
for i in range(NODES // 4):
db.cleanup(NODES + NODES // 2 + i)
assert len(db.kv) == 0
def test_block_18503_changes():
pre = {'0x0c': '0x29d33c02a200937995e632c4597b4dca8e503978'}
toadd = [
['0x', '0x09'],
]
db = RefcountDB(EphemDB())
db.logging = True
NODES = 60
t1 = pruning_trie.Trie(db)
t2 = pruning_trie.Trie(db)
db.ttl = NODES * 2
c = 0
for k, v in pre.items():
triekey = utils.sha3(utils.zpad(utils.decode_hex(k[2:]), 32))
t1.update(triekey, rlp.encode(utils.decode_hex(v[2:])))
t2.update(triekey, rlp.encode(utils.decode_hex(v[2:])))
db.commit_refcount_changes(c)
db.cleanup(c)
c += 1
print(utils.encode_hex(t1.root_hash))
for k, v in toadd:
sys.stderr.write('kv: %s %s\n' % (k, v))
triekey = utils.sha3(utils.zpad(utils.decode_hex(k[2:]), 32))
if v == '0x':
t1.delete(triekey)
else:
t1.update(triekey, rlp.encode(utils.decode_hex(v[2:])))
db.commit_refcount_changes(c)
db.cleanup(c)
c += 1
t1.clear_all()
db.commit_refcount_changes(c)
for i in range(db.ttl + 1):
db.cleanup(c)
c += 1
t3 = pruning_trie.Trie(db)
t3.root_hash = t2.root_hash
print(t3.to_dict())
def test_revert_adds():
db = RefcountDB(EphemDB())
NODES = 60
t1 = pruning_trie.Trie(db)
t2 = pruning_trie.Trie(db)
db.ttl = NODES * 2
for i in range(NODES):
t1.update(to_string(i), to_string(i))
db.commit_refcount_changes(i)
db.cleanup(i)
for i in range(NODES):
t2.update(to_string(i), to_string(i))
db.commit_refcount_changes(NODES + i)
db.cleanup(NODES + i)
for i in range(NODES * 2 - 1, NODES - 1, -1):
db.revert_refcount_changes(i)
for i in range(NODES):
t1.delete(to_string(i))
db.commit_refcount_changes(NODES + i)
db.cleanup(NODES + i)
for i in range(NODES * 2):
db.cleanup(NODES * 2 + i)
assert len(db.kv) == 0
def __init__(self, app):
self.config = app.config
sce = self.config['eth']
if int(sce['pruning']) >= 0:
self.db = RefcountDB(app.services.db)
if "I am not pruning" in self.db.db:
raise Exception("This database was initialized as non-pruning."
" Kinda hard to start pruning now.")
self.db.ttl = int(sce['pruning'])
self.db.db.put("I am pruning", "1")
else:
self.db = app.services.db
if "I am pruning" in self.db:
raise Exception("This database was initialized as pruning."
" Kinda hard to stop pruning now.")
self.db.put("I am not pruning", "1")
if 'network_id' in self.db:
db_network_id = self.db.get('network_id')
if db_network_id != str(sce['network_id']):
raise Exception(
"This database was initialized with network_id {} "
"and can not be used when connecting to network_id {}".
format(db_network_id, sce['network_id']))
else:
self.db.put('network_id', str(sce['network_id']))
self.db.commit()
assert self.db is not None
WiredService.__init__(self, app)
log.info('initializing chain')
coinbase = app.services.accounts.coinbase
env = Env(self.db, sce['block'])
self.chain = Chain(env,
new_head_cb=self._on_new_head,
coinbase=coinbase)
log.info('chain at', number=self.chain.head.number)
if 'genesis_hash' in sce:
assert sce['genesis_hash'] == self.chain.genesis.hex_hash()
self.transaction_queue = Queue(maxsize=self.transaction_queue_size)
self.add_blocks_lock = False
self.add_transaction_lock = gevent.lock.BoundedSemaphore()
self.broadcast_filter = DuplicatesFilter()
self.on_new_head_cbs = []
self.on_new_head_candidate_cbs = []
self.newblock_processing_times = deque(maxlen=1000)
# Consensus
self.consensus_contract = ConsensusContract(
validators=self.config['hdc']['validators'])
self.consensus_manager = ConsensusManager(self,
self.consensus_contract,
self.consensus_privkey)
# lock blocks that where proposed, so they don't get mutated
self.proposal_lock = ProposalLock()
assert not self.proposal_lock.is_locked()
