class BaseCacheStorageTest(unittest.TestCase):
__test__ = False
def setUp(self):
super().setUp()
store = TransactionMemoryStorage()
self.cache_storage = TransactionCacheStorage(store,
self.clock,
capacity=5)
self.cache_storage._manually_initialize()
self.cache_storage.pre_init()
self.genesis = self.cache_storage.get_all_genesis()
self.genesis_blocks = [tx for tx in self.genesis if tx.is_block]
self.genesis_txs = [tx for tx in self.genesis if not tx.is_block]
# Save genesis metadata
self.cache_storage.save_transaction(self.genesis_txs[0],
only_metadata=True)
self.manager = self.create_peer('testnet',
tx_storage=self.cache_storage,
unlock_wallet=True)
def tearDown(self):
super().tearDown()
def _get_new_tx(self, nonce):
tx = Transaction(nonce=nonce, storage=self.cache_storage)
tx.update_hash()
meta = TransactionMetadata(hash=tx.hash)
tx._metadata = meta
return tx
def test_write_read(self):
txs = [self._get_new_tx(nonce) for nonce in range(2 * CACHE_SIZE)]
for tx in txs:
self.cache_storage.save_transaction(tx)
txs2 = [self.cache_storage.get_transaction(tx.hash) for tx in txs]
self.assertEqual(txs, txs2)
def test_dirty_set(self):
txs = [self._get_new_tx(nonce) for nonce in range(CACHE_SIZE)]
for tx in txs:
self.cache_storage.save_transaction(tx)
for tx in txs:
self.assertIn(tx.hash, self.cache_storage.dirty_txs)
# should flush to disk and empty dirty set
self.cache_storage._flush_to_storage(
self.cache_storage.dirty_txs.copy())
self.assertEqual(0, len(self.cache_storage.dirty_txs))
def test_capacity(self):
# cache should not grow over its capacity
txs = [self._get_new_tx(nonce) for nonce in range(2 * CACHE_SIZE)]
for tx in txs:
self.cache_storage.save_transaction(tx)
self.assertEqual(CACHE_SIZE, len(self.cache_storage.cache))
def test_read_adds_to_cache(self):
# make sure reading also adds to cache, not only writes
txs = [self._get_new_tx(nonce) for nonce in range(2 * CACHE_SIZE)]
for tx in txs:
self.cache_storage.save_transaction(tx)
# by now, tx[0] will already have left the cache
self.assertNotIn(txs[0].hash, self.cache_storage.cache)
# read tx
self.cache_storage.get_transaction(txs[0].hash)
# now it should be in cache
self.assertIn(txs[0].hash, self.cache_storage.cache)
def test_read_moves_to_end(self):
# when we read a tx from cache, it should be moved to the end of cache so it's evicted later
txs = [self._get_new_tx(nonce) for nonce in range(2 * CACHE_SIZE)]
for i in range(CACHE_SIZE):
self.cache_storage.save_transaction(txs[i])
# first tx added would be the first to leave cache if we add one more tx
# let's read it from cache so it goes to the end
self.cache_storage.get_transaction(txs[0].hash)
# add a new tx to cache, so it will evict a tx
self.cache_storage.save_transaction(txs[-1])
# first tx should be in cache
self.assertIn(txs[0].hash, self.cache_storage.cache)
def test_cache_eviction(self):
# tests we're evicting the oldest tx from cache
txs = [self._get_new_tx(nonce) for nonce in range(2 * CACHE_SIZE)]
for i in range(CACHE_SIZE):
self.cache_storage.save_transaction(txs[i])
# next save should evict first tx
self.cache_storage.save_transaction(txs[CACHE_SIZE])
self.assertNotIn(txs[0].hash, self.cache_storage.cache)
self.assertIn(txs[CACHE_SIZE].hash, self.cache_storage.cache)
self.assertEqual(CACHE_SIZE, len(self.cache_storage.cache))
def test_flush_thread(self):
txs = [self._get_new_tx(nonce) for nonce in range(CACHE_SIZE)]
for tx in txs:
self.cache_storage.save_transaction(tx)
for tx in txs:
self.assertIn(tx.hash, self.cache_storage.dirty_txs)
# Flush deferred is not None
self.assertIsNotNone(self.cache_storage.flush_deferred)
last_flush_deferred = self.cache_storage.flush_deferred
self.cache_storage._start_flush_thread()
self.assertEqual(last_flush_deferred,
self.cache_storage.flush_deferred)
# We flush the cache and flush_deferred becomes None
self.cache_storage._cb_flush_thread(
self.cache_storage.dirty_txs.copy())
self.assertIsNone(self.cache_storage.flush_deferred)
# After the interval it becomes not None again
self.clock.advance(10)
self.assertIsNotNone(self.cache_storage.flush_deferred)
# If an err occurs, it will become None again and then not None after the interval
self.cache_storage._err_flush_thread('')
self.assertIsNone(self.cache_storage.flush_deferred)
self.clock.advance(5)
self.assertIsNotNone(self.cache_storage.flush_deferred)
# Remove element from cache to test a part of the code
del self.cache_storage.cache[next(iter(self.cache_storage.dirty_txs))]
self.cache_storage._flush_to_storage(
self.cache_storage.dirty_txs.copy())
def test_topological_sort_dfs(self):
_set_test_mode(TestMode.TEST_ALL_WEIGHT)
add_new_blocks(self.manager, 11, advance_clock=1)
tx = add_new_transactions(self.manager, 1, advance_clock=1)[0]
total = 0
for tx in self.cache_storage._topological_sort_dfs(root=tx,
visited=dict()):
total += 1
self.assertEqual(total, 5)
class BasicTransaction(unittest.TestCase):
def setUp(self):
super().setUp()
store = TransactionMemoryStorage()
self.cache_storage = TransactionCacheStorage(store,
self.clock,
capacity=5)
self.cache_storage._manually_initialize()
self.cache_storage.start()
self.genesis = self.cache_storage.get_all_genesis()
self.genesis_blocks = [tx for tx in self.genesis if tx.is_block]
self.genesis_txs = [tx for tx in self.genesis if not tx.is_block]
# Save genesis metadata
self.cache_storage.save_transaction_deferred(self.genesis_txs[0],
only_metadata=True)
self.manager = self.create_peer('testnet',
tx_storage=self.cache_storage,
unlock_wallet=True)
def tearDown(self):
super().tearDown()
def _get_new_tx(self, nonce):
tx = Transaction(nonce=nonce, storage=self.cache_storage)
tx.update_hash()
meta = TransactionMetadata(hash=tx.hash)
tx._metadata = meta
return tx
def test_write_read(self):
txs = [self._get_new_tx(nonce) for nonce in range(2 * CACHE_SIZE)]
for tx in txs:
self.cache_storage.save_transaction(tx)
txs2 = [self.cache_storage.get_transaction(tx.hash) for tx in txs]
self.assertEqual(txs, txs2)
def test_dirty_set(self):
txs = [self._get_new_tx(nonce) for nonce in range(CACHE_SIZE)]
for tx in txs:
self.cache_storage.save_transaction(tx)
for tx in txs:
self.assertIn(tx.hash, self.cache_storage.dirty_txs)
# should flush to disk and empty dirty set
self.cache_storage._flush_to_storage(
self.cache_storage.dirty_txs.copy())
self.assertEqual(0, len(self.cache_storage.dirty_txs))
def test_capacity(self):
# cache should not grow over its capacity
txs = [self._get_new_tx(nonce) for nonce in range(2 * CACHE_SIZE)]
for tx in txs:
self.cache_storage.save_transaction(tx)
self.assertEqual(CACHE_SIZE, len(self.cache_storage.cache))
def test_read_adds_to_cache(self):
# make sure reading also adds to cache, not only writes
txs = [self._get_new_tx(nonce) for nonce in range(2 * CACHE_SIZE)]
for tx in txs:
self.cache_storage.save_transaction(tx)
# by now, tx[0] will already have left the cache
self.assertNotIn(txs[0].hash, self.cache_storage.cache)
# read tx
self.cache_storage.get_transaction(txs[0].hash)
# now it should be in cache
self.assertIn(txs[0].hash, self.cache_storage.cache)
def test_read_moves_to_end(self):
# when we read a tx from cache, it should be moved to the end of cache so it's evicted later
txs = [self._get_new_tx(nonce) for nonce in range(2 * CACHE_SIZE)]
for i in range(CACHE_SIZE):
self.cache_storage.save_transaction(txs[i])
# first tx added would be the first to leave cache if we add one more tx
# let's read it from cache so it goes to the end
self.cache_storage.get_transaction(txs[0].hash)
# add a new tx to cache, so it will evict a tx
self.cache_storage.save_transaction(txs[-1])
# first tx should be in cache
self.assertIn(txs[0].hash, self.cache_storage.cache)
def test_cache_eviction(self):
# tests we're evicting the oldest tx from cache
txs = [self._get_new_tx(nonce) for nonce in range(2 * CACHE_SIZE)]
for i in range(CACHE_SIZE):
self.cache_storage.save_transaction(txs[i])
# next save should evict first tx
self.cache_storage.save_transaction(txs[CACHE_SIZE])
self.assertNotIn(txs[0].hash, self.cache_storage.cache)
self.assertIn(txs[CACHE_SIZE].hash, self.cache_storage.cache)
self.assertEqual(CACHE_SIZE, len(self.cache_storage.cache))
def test_flush_thread(self):
txs = [self._get_new_tx(nonce) for nonce in range(CACHE_SIZE)]
for tx in txs:
self.cache_storage.save_transaction(tx)
for tx in txs:
self.assertIn(tx.hash, self.cache_storage.dirty_txs)
# Flush deferred is not None
self.assertIsNotNone(self.cache_storage.flush_deferred)
last_flush_deferred = self.cache_storage.flush_deferred
self.cache_storage._start_flush_thread()
self.assertEqual(last_flush_deferred,
self.cache_storage.flush_deferred)
# We flush the cache and flush_deferred becomes None
self.cache_storage._cb_flush_thread(
self.cache_storage.dirty_txs.copy())
self.assertIsNone(self.cache_storage.flush_deferred)
# After the interval it becomes not None again
self.clock.advance(10)
self.assertIsNotNone(self.cache_storage.flush_deferred)
# If an err occurs, it will become None again and then not None after the interval
self.cache_storage._err_flush_thread('')
self.assertIsNone(self.cache_storage.flush_deferred)
self.clock.advance(5)
self.assertIsNotNone(self.cache_storage.flush_deferred)
# Remove element from cache to test a part of the code
del self.cache_storage.cache[next(iter(self.cache_storage.dirty_txs))]
self.cache_storage._flush_to_storage(
self.cache_storage.dirty_txs.copy())
def test_deferred_methods(self):
for _ in self._test_deferred_methods():
pass
@inlineCallbacks
def _test_deferred_methods(self):
# Testing without cloning
self.cache_storage._clone_if_needed = False
block_parents = [tx.hash for tx in self.genesis]
output = TxOutput(
200, bytes.fromhex('1e393a5ce2ff1c98d4ff6892f2175100f2dad049'))
obj = Block(timestamp=MIN_TIMESTAMP,
weight=12,
outputs=[output],
parents=block_parents,
nonce=100781,
storage=self.cache_storage)
obj.resolve()
self.cache_storage.save_transaction_deferred(obj)
loaded_obj1 = yield self.cache_storage.get_transaction_deferred(
obj.hash)
metadata_obj1_def = yield self.cache_storage.get_metadata_deferred(
obj.hash)
metadata_obj1 = obj.get_metadata()
self.assertEqual(metadata_obj1_def, metadata_obj1)
metadata_error = yield self.cache_storage.get_metadata_deferred(
bytes.fromhex(
'0001569c85fffa5782c3979e7d68dce1d8d84772505a53ddd76d636585f3977e'
))
self.assertIsNone(metadata_error)
self.cache_storage._flush_to_storage(
self.cache_storage.dirty_txs.copy())
self.cache_storage.cache = collections.OrderedDict()
loaded_obj2 = yield self.cache_storage.get_transaction_deferred(
obj.hash)
self.assertEqual(loaded_obj1, loaded_obj2)
self.assertTrue(
(yield self.cache_storage.transaction_exists_deferred(obj.hash)))
self.assertFalse((yield self.cache_storage.transaction_exists_deferred(
'0001569c85fffa5782c3979e7d68dce1d8d84772505a53ddd76d636585f3977e')
))
self.assertFalse(
self.cache_storage.transaction_exists(
'0001569c85fffa5782c3979e7d68dce1d8d84772505a53ddd76d636585f3977e'
))
self.assertEqual(obj, loaded_obj1)
self.assertEqual(obj.is_block, loaded_obj1.is_block)
count = yield self.cache_storage.get_count_tx_blocks_deferred()
self.assertEqual(count, 4)
all_transactions = yield self.cache_storage.get_all_transactions_deferred(
)
total = 0
for tx in all_transactions:
total += 1
self.assertEqual(total, 4)
def test_topological_sort_dfs(self):
self.manager.test_mode = TestMode.TEST_ALL_WEIGHT
add_new_blocks(self.manager, 11, advance_clock=1)
tx = add_new_transactions(self.manager, 1, advance_clock=1)[0]
total = 0
for tx in self.cache_storage._topological_sort_dfs(root=tx,
visited=dict()):
total += 1
self.assertEqual(total, 5)