def test_two_trees(): db = RefcountDB(EphemDB()) NODES = 60 t1 = pruning_trie.Trie(db) t2 = pruning_trie.Trie(db) db.ttl = 0 for i in range(NODES): t1.update(to_string(i), to_string(i)) if i < NODES // 2: t2.update(to_string(i), to_string(i)) db.commit_refcount_changes(i) db.cleanup(i) check_db_tightness([t1, t2], db) for i in range(NODES): sys.stderr.write('clearing: %d\n' % i) t1.delete(to_string(NODES - 1 - i)) db.commit_refcount_changes(NODES + i) db.cleanup(NODES + i) check_db_tightness([t1, t2], db) assert t2.to_dict() == {to_string(i): to_string(i) for i in range(NODES // 2)} for i in range(NODES // 2): t2.delete(to_string(i)) db.commit_refcount_changes(NODES * 2 + i) db.cleanup(NODES * 2 + i) check_db_tightness([t1, t2], db) assert len(db.kv) == 0
def __init__(self, app): self.config = app.config sce = self.config['eth'] if int(sce['pruning']) >= 0: self.db = RefcountDB(app.services.db) if "I am not pruning" in self.db.db: raise RuntimeError( "The database in '{}' was initialized as non-pruning. " "Can not enable pruning now.".format( self.config['data_dir'])) self.db.ttl = int(sce['pruning']) self.db.db.put("I am pruning", "1") else: self.db = app.services.db if "I am pruning" in self.db: raise RuntimeError( "The database in '{}' was initialized as pruning. " "Can not disable pruning now".format( self.config['data_dir'])) self.db.put("I am not pruning", "1") if 'network_id' in self.db: db_network_id = self.db.get('network_id') if db_network_id != str(sce['network_id']): raise RuntimeError( "The database in '{}' was initialized with network id {} and can not be used " "when connecting to network id {}. Please choose a different data directory." .format(self.config['data_dir'], db_network_id, sce['network_id'])) else: self.db.put('network_id', str(sce['network_id'])) self.db.commit() assert self.db is not None super(ChainService, self).__init__(app) log.info('initializing chain') coinbase = app.services.accounts.coinbase env = Env(self.db, sce['block']) self.chain = Chain(env, new_head_cb=self._on_new_head, coinbase=coinbase) log.info('chain at', number=self.chain.head.number) if 'genesis_hash' in sce: assert sce['genesis_hash'] == self.chain.genesis.hex_hash(), \ "Genesis hash mismatch.\n Expected: %s\n Got: %s" % ( sce['genesis_hash'], self.chain.genesis.hex_hash()) self.synchronizer = Synchronizer(self, force_sync=None) self.block_queue = Queue(maxsize=self.block_queue_size) self.transaction_queue = Queue(maxsize=self.transaction_queue_size) self.add_blocks_lock = False self.add_transaction_lock = gevent.lock.Semaphore() self.broadcast_filter = DuplicatesFilter() self.on_new_head_cbs = [] self.on_new_head_candidate_cbs = [] self.newblock_processing_times = deque(maxlen=1000)
def test_basic_pruning(): db = RefcountDB(EphemDB()) NODES = 60 t = pruning_trie.Trie(db) db.ttl = 0 db.logging = True for i in range(NODES): t.update(to_string(i), to_string(i)) db.commit_refcount_changes(0) db.cleanup(0) check_db_tightness([t], db) for i in range(NODES): t.update(to_string(i), to_string(i ** 3)) db.commit_refcount_changes(0) db.cleanup(0) check_db_tightness([t], db) for i in range(NODES): t.delete(to_string(i)) db.commit_refcount_changes(0) db.cleanup(0) check_db_tightness([t], db) assert len(t.to_dict()) == 0 assert len(db.kv) == 0
def run_test(name): pairs = load_tests()[name] def _dec(x): if utils.is_string(x) and x.startswith(b'0x'): return utils.decode_hex(x[2:]) return x pairs['in'] = [(_dec(k), _dec(v)) for k, v in pairs['in']] deletes = [(k, v) for k, v in pairs['in'] if v is None] N_PERMUTATIONS = 100 for i, permut in enumerate(itertools.permutations(pairs['in'])): if i > N_PERMUTATIONS: break db = RefcountDB(EphemDB()) db.ttl = 0 t = pruning_trie.Trie(db) for k, v in permut: # logger.debug('updating with (%s, %s)' %(k, v)) if v is not None: t.update(k, v) else: t.delete(k) db.commit_refcount_changes(0) db.cleanup(0) # make sure we have deletes at the end for k, v in deletes: t.delete(k) t.clear_all() db.commit_refcount_changes(1) db.cleanup(1) assert len(db.kv) == 0 assert pairs['root'] == b'0x' + utils.encode_hex(t.root_hash), (i, list(permut) + deletes)
def test_deep_inner_branch_deletion(): db = RefcountDB(EphemDB()) db.logging = True db.ttl = 1 t1 = pruning_trie.Trie(db) t1.update(b'etherdogecoin', b'\x33' * 50) t1.update(b'etherdogelot', b'\x44' * 50) t1.delete(b'etherhouse') t1.delete(b'etherhouse') t1.delete(b'etherhouse') t1.delete(b'etherhouse')
def __init__(self, app): self.config = app.config sce = self.config['eth'] if int(sce['pruning']) >= 0: self.db = RefcountDB(app.services.db) if "I am not pruning" in self.db.db: raise Exception("This database was initialized as non-pruning." " Kinda hard to start pruning now.") self.db.ttl = int(sce['pruning']) self.db.db.put("I am pruning", "1") else: self.db = app.services.db if "I am pruning" in self.db: raise Exception("This database was initialized as pruning." " Kinda hard to stop pruning now.") self.db.put("I am not pruning", "1") if 'network_id' in self.db: db_network_id = self.db.get('network_id') if db_network_id != str(sce['network_id']): raise Exception( "This database was initialized with network_id {} " "and can not be used when connecting to network_id {}". format(db_network_id, sce['network_id'])) else: self.db.put('network_id', str(sce['network_id'])) self.db.commit() assert self.db is not None super(ChainService, self).__init__(app) log.info('initializing chain') coinbase = app.services.accounts.coinbase env = Env(self.db, sce['block']) self.chain = Chain(env, new_head_cb=self._on_new_head, coinbase=coinbase) log.info('chain at', number=self.chain.head.number) if 'genesis_hash' in sce: assert sce['genesis_hash'] == self.chain.genesis.hex_hash() self.synchronizer = Synchronizer(self, force_sync=None) self.block_queue = Queue(maxsize=self.block_queue_size) self.transaction_queue = Queue(maxsize=self.transaction_queue_size) self.add_blocks_lock = False self.add_transaction_lock = gevent.lock.Semaphore() self.broadcast_filter = DuplicatesFilter() self.on_new_head_cbs = [] self.on_new_head_candidate_cbs = [] self.newblock_processing_times = deque(maxlen=1000)
def test_clear(): db = RefcountDB(EphemDB()) NODES = 60 t = pruning_trie.Trie(db) db.ttl = 0 for i in range(NODES): t.update(to_string(i), to_string(i)) db.commit_refcount_changes(i) db.cleanup(i) t.clear_all() db.commit_refcount_changes(NODES) db.cleanup(NODES) assert len(db.kv) == 0
def test_two_tries_with_small_root_node(): db = RefcountDB(EphemDB()) db.logging = True db.ttl = 1 t1 = pruning_trie.Trie(db) t2 = pruning_trie.Trie(db) t1.update(b'3', b'5') t2.update(b'3', b'5') t1.delete(b'3') db.commit_refcount_changes(0) db.cleanup(0) db.cleanup(1) db.cleanup(2) print(db.kv) print(t2.to_dict())
def test_insert_delete(): for a in (5, 15, 60): db = RefcountDB(EphemDB()) NODES = a t1 = pruning_trie.Trie(db) db.ttl = 0 db.logging = True for i in range(NODES): t1.update(to_string(i), to_string(i)) db.commit_refcount_changes(i) db.cleanup(i) check_db_tightness([t1], db) for i in range(NODES): t1.delete(to_string(NODES - 1 - i)) db.commit_refcount_changes(NODES + i) db.cleanup(NODES + i) check_db_tightness([t1], db) assert len(db.kv) == 0
def test_trie_transfer(): db = RefcountDB(EphemDB()) NODES = 60 t1 = pruning_trie.Trie(db) db.ttl = NODES * 2 for i in range(NODES): t1.update(to_string(i), to_string(i)) db.commit_refcount_changes(i) db.cleanup(i) t2 = pruning_trie.Trie(db) t2.root_hash = t1.root_hash assert t2.to_dict() == {to_string(i): to_string(i) for i in range(NODES)} for i in range(NODES): t2.delete(to_string(i)) db.commit_refcount_changes(NODES + i) db.cleanup(NODES + i) for i in range(NODES * 2): db.cleanup(2 * NODES + i) assert len(db.kv) == 0
def __init__(self, app): self.config = app.config sce = self.config['eth'] if int(sce['pruning']) >= 0: self.db = RefcountDB(app.services.db) if "I am not pruning" in self.db.db: raise Exception("This database was initialized as non-pruning." " Kinda hard to start pruning now.") self.db.ttl = int(sce['pruning']) self.db.db.put("I am pruning", "1") else: self.db = app.services.db if "I am pruning" in self.db: raise Exception("This database was initialized as pruning." " Kinda hard to stop pruning now.") self.db.put("I am not pruning", "1") assert self.db is not None super(ChainService, self).__init__(app) log.info('initializing chain') coinbase = app.services.accounts.coinbase if sce['genesis']: log.info('loading genesis', path=sce['genesis']) _json = json.load(open(sce['genesis'])) else: log.info('loaded default genesis alloc') _json = None _genesis = genesis(self.db, json=_json) log.info('created genesis block', hash=encode_hex(_genesis.hash)) self.chain = Chain(self.db, genesis=_genesis, new_head_cb=self._on_new_head, coinbase=coinbase) log.info('chain at', number=self.chain.head.number) self.synchronizer = Synchronizer(self, force_sync=None) self.block_queue = Queue(maxsize=self.block_queue_size) self.transaction_queue = Queue(maxsize=self.transaction_queue_size) self.add_blocks_lock = False self.add_transaction_lock = gevent.lock.Semaphore() self.broadcast_filter = DuplicatesFilter() self.on_new_head_cbs = [] self.on_new_head_candidate_cbs = [] self.newblock_processing_times = deque(maxlen=1000)
def test_block_18315_changes(): pre = {} toadd = [ ['0x0000000000000000000000000000000000000000000000000000000000000000', '0xf9e88bc2b3203e764fe67b4d0f4171b7756117c8'], ['0x0000000000000000000000000000000000000000000000000000000000000001', '0x'], ['0x0000000000000000000000000000000000000000000000000000000000000002', '0x'], ] db = RefcountDB(EphemDB()) db.logging = True NODES = 60 t1 = pruning_trie.Trie(db) t2 = pruning_trie.Trie(db) db.ttl = NODES * 2 c = 0 for k, v in pre.items(): triekey = utils.sha3(utils.zpad(k[2:].decode('hex'), 32)) t1.update(triekey, rlp.encode(v[2:].decode('hex'))) t2.update(triekey, rlp.encode(v[2:].decode('hex'))) db.commit_refcount_changes(c) db.cleanup(c) c += 1 sys.stderr.write('##############################\n') print(utils.encode_hex(t1.root_hash)) print(t1.to_dict()) for k, v in toadd: sys.stderr.write('kv: %s %s\n' % (k, v)) triekey = utils.sha3(utils.zpad(utils.decode_hex(k[2:]), 32)) if v == '0x': t1.delete(triekey) else: t1.update(triekey, rlp.encode(utils.decode_hex(v[2:]))) db.commit_refcount_changes(c) db.cleanup(c) c += 1 t1.clear_all() db.commit_refcount_changes(c) for i in range(db.ttl + 1): db.cleanup(c) c += 1 t3 = pruning_trie.Trie(db) t3.root_hash = t2.root_hash print(t3.to_dict())
def test_shared_prefix(): db = RefcountDB(EphemDB()) db.logging = True db.ttl = 1 t1 = pruning_trie.Trie(db) t2 = pruning_trie.Trie(db) t1.update(b'dogecoin', b'\x33' * 50) t1.update(b'dogelot', b'\x44' * 50) t2.update(b'dogecoin', b'\x33' * 50) t2.update(b'dogelot', b'\x44' * 50) print(db.kv) t1.delete(b'dogecoin') t1.delete(b'dogelot') print(db.kv) db.commit_refcount_changes(0) db.cleanup(0) db.cleanup(1) db.cleanup(2) print(db.kv) print(t2.to_dict())
def test_revert_deletes(): db = RefcountDB(EphemDB()) NODES = 60 t1 = pruning_trie.Trie(db) db.ttl = NODES * 2 for i in range(NODES): t1.update(to_string(i), to_string(i)) db.commit_refcount_changes(i) db.cleanup(i) x = t1.root_hash for i in range(NODES): t1.delete(to_string(i)) db.commit_refcount_changes(NODES + i) db.cleanup(NODES + i) for i in range(NODES * 2 - 1, NODES - 1, -1): db.revert_refcount_changes(i) for i in range(NODES * 2): db.cleanup(NODES + i) db.revert_refcount_changes(i) t1.root_hash = x assert t1.to_dict() == {to_string(i): to_string(i) for i in range(NODES)}
def test_delayed_pruning(): NODES = 60 db = RefcountDB(EphemDB()) t = pruning_trie.Trie(db) db.ttl = NODES // 4 for i in range(NODES): t.update(to_string(i), to_string(i)) db.commit_refcount_changes(i) db.cleanup(i) for i in range(NODES): t.update(to_string(i), to_string(i ** 3)) db.commit_refcount_changes(i + NODES) db.cleanup(i + NODES) for i in range(NODES): t.delete(to_string(i)) db.commit_refcount_changes(i + NODES * 2) db.cleanup(i + NODES * 2) for i in range(NODES // 4): db.cleanup(i + NODES * 3) assert len(t.to_dict()) == 0 assert len(db.kv) == 0
def test_two_trees_with_clear(): db = RefcountDB(EphemDB()) NODES = 60 t1 = pruning_trie.Trie(db) t2 = pruning_trie.Trie(db) db.ttl = NODES // 4 for i in range(NODES): t1.update(to_string(i), to_string(i)) if i < NODES // 2: t2.update(to_string(i), to_string(i)) db.commit_refcount_changes(i) db.cleanup(i) t1.clear_all() db.cleanup(NODES) assert t2.to_dict() == {to_string(i): to_string(i) for i in range(NODES // 2)} for i in range(NODES // 2): t2.delete(to_string(i)) db.commit_refcount_changes(NODES + i) db.cleanup(NODES + i) for i in range(NODES // 4): db.cleanup(NODES + NODES // 2 + i) assert len(db.kv) == 0
def test_block_18503_changes(): pre = {'0x0c': '0x29d33c02a200937995e632c4597b4dca8e503978'} toadd = [ ['0x', '0x09'], ] db = RefcountDB(EphemDB()) db.logging = True NODES = 60 t1 = pruning_trie.Trie(db) t2 = pruning_trie.Trie(db) db.ttl = NODES * 2 c = 0 for k, v in pre.items(): triekey = utils.sha3(utils.zpad(utils.decode_hex(k[2:]), 32)) t1.update(triekey, rlp.encode(utils.decode_hex(v[2:]))) t2.update(triekey, rlp.encode(utils.decode_hex(v[2:]))) db.commit_refcount_changes(c) db.cleanup(c) c += 1 print(utils.encode_hex(t1.root_hash)) for k, v in toadd: sys.stderr.write('kv: %s %s\n' % (k, v)) triekey = utils.sha3(utils.zpad(utils.decode_hex(k[2:]), 32)) if v == '0x': t1.delete(triekey) else: t1.update(triekey, rlp.encode(utils.decode_hex(v[2:]))) db.commit_refcount_changes(c) db.cleanup(c) c += 1 t1.clear_all() db.commit_refcount_changes(c) for i in range(db.ttl + 1): db.cleanup(c) c += 1 t3 = pruning_trie.Trie(db) t3.root_hash = t2.root_hash print(t3.to_dict())
def test_revert_adds(): db = RefcountDB(EphemDB()) NODES = 60 t1 = pruning_trie.Trie(db) t2 = pruning_trie.Trie(db) db.ttl = NODES * 2 for i in range(NODES): t1.update(to_string(i), to_string(i)) db.commit_refcount_changes(i) db.cleanup(i) for i in range(NODES): t2.update(to_string(i), to_string(i)) db.commit_refcount_changes(NODES + i) db.cleanup(NODES + i) for i in range(NODES * 2 - 1, NODES - 1, -1): db.revert_refcount_changes(i) for i in range(NODES): t1.delete(to_string(i)) db.commit_refcount_changes(NODES + i) db.cleanup(NODES + i) for i in range(NODES * 2): db.cleanup(NODES * 2 + i) assert len(db.kv) == 0
def __init__(self, app): self.config = app.config sce = self.config['eth'] if int(sce['pruning']) >= 0: self.db = RefcountDB(app.services.db) if "I am not pruning" in self.db.db: raise Exception("This database was initialized as non-pruning." " Kinda hard to start pruning now.") self.db.ttl = int(sce['pruning']) self.db.db.put("I am pruning", "1") else: self.db = app.services.db if "I am pruning" in self.db: raise Exception("This database was initialized as pruning." " Kinda hard to stop pruning now.") self.db.put("I am not pruning", "1") if 'network_id' in self.db: db_network_id = self.db.get('network_id') if db_network_id != str(sce['network_id']): raise Exception( "This database was initialized with network_id {} " "and can not be used when connecting to network_id {}". format(db_network_id, sce['network_id'])) else: self.db.put('network_id', str(sce['network_id'])) self.db.commit() assert self.db is not None WiredService.__init__(self, app) log.info('initializing chain') coinbase = app.services.accounts.coinbase env = Env(self.db, sce['block']) self.chain = Chain(env, new_head_cb=self._on_new_head, coinbase=coinbase) log.info('chain at', number=self.chain.head.number) if 'genesis_hash' in sce: assert sce['genesis_hash'] == self.chain.genesis.hex_hash() self.transaction_queue = Queue(maxsize=self.transaction_queue_size) self.add_blocks_lock = False self.add_transaction_lock = gevent.lock.BoundedSemaphore() self.broadcast_filter = DuplicatesFilter() self.on_new_head_cbs = [] self.on_new_head_candidate_cbs = [] self.newblock_processing_times = deque(maxlen=1000) # Consensus self.consensus_contract = ConsensusContract( validators=self.config['hdc']['validators']) self.consensus_manager = ConsensusManager(self, self.consensus_contract, self.consensus_privkey) # lock blocks that where proposed, so they don't get mutated self.proposal_lock = ProposalLock() assert not self.proposal_lock.is_locked()