def save_transaction(self,
tx: 'BaseTransaction',
*,
only_metadata: bool = False) -> None:
skip_warning(super().save_transaction)(tx, only_metadata=only_metadata)
if tx.is_genesis:
return
self._save_transaction(tx, only_metadata=only_metadata)
self._save_to_weakref(tx)
def remove_transaction(self, tx: 'BaseTransaction') -> None:
assert tx.hash is not None
skip_warning(super().remove_transaction)(tx)
filepath = self.generate_filepath(tx.hash)
self._remove_from_weakref(tx)
try:
os.unlink(filepath)
except FileNotFoundError:
pass
def _flush_to_storage(self, dirty_txs_copy: Set[bytes]) -> None:
"""Write dirty pages to disk."""
for tx_hash in dirty_txs_copy:
# a dirty tx might be removed from self.cache outside this thread: if _update_cache is called
# and we need to save the tx to disk immediately. So it might happen that the tx which was
# in the dirty set when the flush thread began is not in cache anymore, hence this `if` check
if tx_hash in self.cache:
tx = self._clone(self.cache[tx_hash])
self.dirty_txs.discard(tx_hash)
skip_warning(self.store._save_transaction)(tx)
def Count(self, request: protos.CountRequest, context: _Context) -> protos.CountResponse:
tx_type = request.tx_type
if tx_type == protos.ANY_TYPE:
count = skip_warning(self.storage.get_count_tx_blocks)()
elif tx_type == protos.TRANSACTION_TYPE:
count = skip_warning(self.storage.get_tx_count)()
elif tx_type == protos.BLOCK_TYPE:
count = skip_warning(self.storage.get_block_count)()
else:
raise ValueError('invalid tx_type %s' % (tx_type,))
return protos.CountResponse(count=count)
def Remove(self, request: protos.RemoveRequest, context: _Context) -> protos.RemoveResponse:
from hathor.transaction import tx_or_block_from_proto
tx_proto = request.transaction
result = protos.RemoveResponse(removed=False)
tx = tx_or_block_from_proto(tx_proto, storage=self.storage)
skip_warning(self.storage.remove_transaction)(tx)
result.removed = True
return result
def Save(self, request: protos.SaveRequest, context: _Context) -> protos.SaveResponse:
from hathor.transaction import tx_or_block_from_proto
tx_proto = request.transaction
only_metadata = request.only_metadata
result = protos.SaveResponse(saved=False)
tx = tx_or_block_from_proto(tx_proto, storage=self.storage)
skip_warning(self.storage.save_transaction)(tx, only_metadata=only_metadata)
result.saved = True
return result
def save_transaction_deferred(self,
tx: BaseTransaction,
*,
only_metadata: bool = False) -> Deferred:
return succeed(
skip_warning(self.save_transaction)(tx,
only_metadata=only_metadata))
def List(self, request: protos.ListRequest, context: _Context) -> Iterator[protos.ListItemResponse]:
exclude_metadata = request.exclude_metadata
has_more = None
hash_bytes = request.hash
count = request.max_count
timestamp = request.timestamp
# TODO: more exceptions for unsupported cases
if request.filter_before:
if request.tx_type == protos.ANY_TYPE:
raise NotImplementedError
elif request.tx_type == protos.TRANSACTION_TYPE:
tx_iter = self.storage.get_transactions_before(hash_bytes, count)
elif request.tx_type == protos.BLOCK_TYPE:
tx_iter = self.storage.get_blocks_before(hash_bytes, count)
else:
raise ValueError('invalid tx_type %s' % (request.tx_type,))
elif request.time_filter == protos.ONLY_NEWER:
if request.tx_type == protos.ANY_TYPE:
raise NotImplementedError
elif request.tx_type == protos.TRANSACTION_TYPE:
tx_iter, has_more = self.storage.get_newer_txs_after(timestamp, hash_bytes, count)
elif request.tx_type == protos.BLOCK_TYPE:
tx_iter, has_more = self.storage.get_newer_blocks_after(timestamp, hash_bytes, count)
else:
raise ValueError('invalid tx_type %s' % (request.tx_type,))
elif request.time_filter == protos.ONLY_OLDER:
if request.tx_type == protos.ANY_TYPE:
raise NotImplementedError
elif request.tx_type == protos.TRANSACTION_TYPE:
tx_iter, has_more = self.storage.get_older_txs_after(timestamp, hash_bytes, count)
elif request.tx_type == protos.BLOCK_TYPE:
tx_iter, has_more = self.storage.get_older_blocks_after(timestamp, hash_bytes, count)
else:
raise ValueError('invalid tx_type %s' % (request.tx_type,))
elif request.time_filter == protos.NO_FILTER:
if request.order_by == protos.ANY_ORDER:
tx_iter = skip_warning(self.storage.get_all_transactions)()
elif request.order_by == protos.TOPOLOGICAL_ORDER:
tx_iter = self.storage._topological_sort()
else:
raise ValueError('invalid order_by')
else:
raise ValueError('invalid request')
for tx in tx_iter:
if exclude_metadata:
del tx._metadata
else:
tx.get_metadata()
yield protos.ListItemResponse(transaction=tx.to_proto())
if has_more is not None:
yield protos.ListItemResponse(has_more=has_more)
def _update_cache(self, tx: BaseTransaction) -> None:
"""Updates the cache making sure it has at most the number of elements configured
as its capacity.
If we need to evict a tx from cache and it's dirty, write it to disk immediately.
"""
assert tx.hash is not None
_tx = self.cache.get(tx.hash, None)
if not _tx:
if len(self.cache) >= self.capacity:
(_, removed_tx) = self.cache.popitem(last=False)
if removed_tx.hash in self.dirty_txs:
# write to disk so we don't lose the last update
self.dirty_txs.discard(removed_tx.hash)
skip_warning(self.store.save_transaction)(removed_tx)
self.cache[tx.hash] = self._clone(tx)
else:
# Tx might have been updated
self.cache[tx.hash] = self._clone(tx)
self.cache.move_to_end(tx.hash, last=True)
def Get(self, request: protos.GetRequest, context: _Context) -> protos.GetResponse:
hash_bytes = request.hash
exclude_metadata = request.exclude_metadata
tx = skip_warning(self.storage.get_transaction)(hash_bytes)
if exclude_metadata:
del tx._metadata
else:
tx.get_metadata()
return protos.GetResponse(transaction=tx.to_proto())
def get_transaction(self, hash_bytes: bytes) -> BaseTransaction:
if hash_bytes in self.cache:
tx = self._clone(self.cache[hash_bytes])
self.cache.move_to_end(hash_bytes, last=True)
self.stats['hit'] += 1
else:
tx = skip_warning(self.store.get_transaction)(hash_bytes)
tx.storage = self
self._update_cache(tx)
self.stats['miss'] += 1
self._save_to_weakref(tx)
return tx
def get_all_transactions(self):
self._flush_to_storage(self.dirty_txs.copy())
for tx in skip_warning(self.store.get_all_transactions)():
tx.storage = self
self._save_to_weakref(tx)
yield tx
def save_transaction(self,
tx: BaseTransaction,
*,
only_metadata: bool = False) -> None:
skip_warning(super().save_transaction)(tx, only_metadata=only_metadata)
self._save_transaction(tx, only_metadata=only_metadata)
def get_all_transactions_deferred(self) -> Deferred:
return succeed(skip_warning(self.get_all_transactions)())
def transaction_exists_deferred(self, hash_bytes: bytes) -> Deferred:
return succeed(skip_warning(self.transaction_exists)(hash_bytes))
def transaction_exists(self, hash_bytes: bytes) -> bool:
if hash_bytes in self.cache:
return True
return skip_warning(self.store.transaction_exists)(hash_bytes)
def save_transaction(self, tx: BaseTransaction, *, only_metadata: bool = False) -> None:
self._save_transaction(tx)
self._save_to_weakref(tx)
# call super which adds to index if needed
skip_warning(super().save_transaction)(tx, only_metadata=only_metadata)
def remove_transaction(self, tx: BaseTransaction) -> None:
assert tx.hash is not None
skip_warning(super().remove_transaction)(tx)
self.cache.pop(tx.hash, None)
self.dirty_txs.discard(tx.hash)
self._remove_from_weakref(tx)
def remove_transaction(self, tx: 'BaseTransaction') -> None:
skip_warning(super().remove_transaction)(tx)
self._db.delete(tx.hash)
self._remove_from_weakref(tx)
def remove_transaction_deferred(self, tx: BaseTransaction) -> Deferred:
return succeed(skip_warning(self.remove_transaction)(tx))
def get_count_tx_blocks(self) -> int:
self._flush_to_storage(self.dirty_txs.copy())
return skip_warning(self.store.get_count_tx_blocks)()
def get_transaction_deferred(self, hash_bytes: bytes) -> Deferred:
return succeed(skip_warning(self.get_transaction)(hash_bytes))
def Exists(self, request: protos.ExistsRequest, context: _Context) -> protos.ExistsResponse:
hash_bytes = request.hash
exists = skip_warning(self.storage.transaction_exists)(hash_bytes)
return protos.ExistsResponse(exists=exists)
def get_count_tx_blocks_deferred(self) -> Deferred:
return succeed(skip_warning(self.get_count_tx_blocks)())
def remove_transaction(self, tx: BaseTransaction) -> None:
assert tx.hash is not None
skip_warning(super().remove_transaction)(tx)
self.transactions.pop(tx.hash, None)
self.metadata.pop(tx.hash, None)
