def save_transaction(self, tx: 'BaseTransaction', *, only_metadata: bool = False) -> None: skip_warning(super().save_transaction)(tx, only_metadata=only_metadata) if tx.is_genesis: return self._save_transaction(tx, only_metadata=only_metadata) self._save_to_weakref(tx)
def remove_transaction(self, tx: 'BaseTransaction') -> None: assert tx.hash is not None skip_warning(super().remove_transaction)(tx) filepath = self.generate_filepath(tx.hash) self._remove_from_weakref(tx) try: os.unlink(filepath) except FileNotFoundError: pass
def _flush_to_storage(self, dirty_txs_copy: Set[bytes]) -> None: """Write dirty pages to disk.""" for tx_hash in dirty_txs_copy: # a dirty tx might be removed from self.cache outside this thread: if _update_cache is called # and we need to save the tx to disk immediately. So it might happen that the tx which was # in the dirty set when the flush thread began is not in cache anymore, hence this `if` check if tx_hash in self.cache: tx = self._clone(self.cache[tx_hash]) self.dirty_txs.discard(tx_hash) skip_warning(self.store._save_transaction)(tx)
def Count(self, request: protos.CountRequest, context: _Context) -> protos.CountResponse: tx_type = request.tx_type if tx_type == protos.ANY_TYPE: count = skip_warning(self.storage.get_count_tx_blocks)() elif tx_type == protos.TRANSACTION_TYPE: count = skip_warning(self.storage.get_tx_count)() elif tx_type == protos.BLOCK_TYPE: count = skip_warning(self.storage.get_block_count)() else: raise ValueError('invalid tx_type %s' % (tx_type,)) return protos.CountResponse(count=count)
def Remove(self, request: protos.RemoveRequest, context: _Context) -> protos.RemoveResponse: from hathor.transaction import tx_or_block_from_proto tx_proto = request.transaction result = protos.RemoveResponse(removed=False) tx = tx_or_block_from_proto(tx_proto, storage=self.storage) skip_warning(self.storage.remove_transaction)(tx) result.removed = True return result
def Save(self, request: protos.SaveRequest, context: _Context) -> protos.SaveResponse: from hathor.transaction import tx_or_block_from_proto tx_proto = request.transaction only_metadata = request.only_metadata result = protos.SaveResponse(saved=False) tx = tx_or_block_from_proto(tx_proto, storage=self.storage) skip_warning(self.storage.save_transaction)(tx, only_metadata=only_metadata) result.saved = True return result
def save_transaction_deferred(self, tx: BaseTransaction, *, only_metadata: bool = False) -> Deferred: return succeed( skip_warning(self.save_transaction)(tx, only_metadata=only_metadata))
def List(self, request: protos.ListRequest, context: _Context) -> Iterator[protos.ListItemResponse]: exclude_metadata = request.exclude_metadata has_more = None hash_bytes = request.hash count = request.max_count timestamp = request.timestamp # TODO: more exceptions for unsupported cases if request.filter_before: if request.tx_type == protos.ANY_TYPE: raise NotImplementedError elif request.tx_type == protos.TRANSACTION_TYPE: tx_iter = self.storage.get_transactions_before(hash_bytes, count) elif request.tx_type == protos.BLOCK_TYPE: tx_iter = self.storage.get_blocks_before(hash_bytes, count) else: raise ValueError('invalid tx_type %s' % (request.tx_type,)) elif request.time_filter == protos.ONLY_NEWER: if request.tx_type == protos.ANY_TYPE: raise NotImplementedError elif request.tx_type == protos.TRANSACTION_TYPE: tx_iter, has_more = self.storage.get_newer_txs_after(timestamp, hash_bytes, count) elif request.tx_type == protos.BLOCK_TYPE: tx_iter, has_more = self.storage.get_newer_blocks_after(timestamp, hash_bytes, count) else: raise ValueError('invalid tx_type %s' % (request.tx_type,)) elif request.time_filter == protos.ONLY_OLDER: if request.tx_type == protos.ANY_TYPE: raise NotImplementedError elif request.tx_type == protos.TRANSACTION_TYPE: tx_iter, has_more = self.storage.get_older_txs_after(timestamp, hash_bytes, count) elif request.tx_type == protos.BLOCK_TYPE: tx_iter, has_more = self.storage.get_older_blocks_after(timestamp, hash_bytes, count) else: raise ValueError('invalid tx_type %s' % (request.tx_type,)) elif request.time_filter == protos.NO_FILTER: if request.order_by == protos.ANY_ORDER: tx_iter = skip_warning(self.storage.get_all_transactions)() elif request.order_by == protos.TOPOLOGICAL_ORDER: tx_iter = self.storage._topological_sort() else: raise ValueError('invalid order_by') else: raise ValueError('invalid request') for tx in tx_iter: if exclude_metadata: del tx._metadata else: tx.get_metadata() yield protos.ListItemResponse(transaction=tx.to_proto()) if has_more is not None: yield protos.ListItemResponse(has_more=has_more)
def _update_cache(self, tx: BaseTransaction) -> None: """Updates the cache making sure it has at most the number of elements configured as its capacity. If we need to evict a tx from cache and it's dirty, write it to disk immediately. """ assert tx.hash is not None _tx = self.cache.get(tx.hash, None) if not _tx: if len(self.cache) >= self.capacity: (_, removed_tx) = self.cache.popitem(last=False) if removed_tx.hash in self.dirty_txs: # write to disk so we don't lose the last update self.dirty_txs.discard(removed_tx.hash) skip_warning(self.store.save_transaction)(removed_tx) self.cache[tx.hash] = self._clone(tx) else: # Tx might have been updated self.cache[tx.hash] = self._clone(tx) self.cache.move_to_end(tx.hash, last=True)
def Get(self, request: protos.GetRequest, context: _Context) -> protos.GetResponse: hash_bytes = request.hash exclude_metadata = request.exclude_metadata tx = skip_warning(self.storage.get_transaction)(hash_bytes) if exclude_metadata: del tx._metadata else: tx.get_metadata() return protos.GetResponse(transaction=tx.to_proto())
def get_transaction(self, hash_bytes: bytes) -> BaseTransaction: if hash_bytes in self.cache: tx = self._clone(self.cache[hash_bytes]) self.cache.move_to_end(hash_bytes, last=True) self.stats['hit'] += 1 else: tx = skip_warning(self.store.get_transaction)(hash_bytes) tx.storage = self self._update_cache(tx) self.stats['miss'] += 1 self._save_to_weakref(tx) return tx
def get_all_transactions(self): self._flush_to_storage(self.dirty_txs.copy()) for tx in skip_warning(self.store.get_all_transactions)(): tx.storage = self self._save_to_weakref(tx) yield tx
def save_transaction(self, tx: BaseTransaction, *, only_metadata: bool = False) -> None: skip_warning(super().save_transaction)(tx, only_metadata=only_metadata) self._save_transaction(tx, only_metadata=only_metadata)
def get_all_transactions_deferred(self) -> Deferred: return succeed(skip_warning(self.get_all_transactions)())
def transaction_exists_deferred(self, hash_bytes: bytes) -> Deferred: return succeed(skip_warning(self.transaction_exists)(hash_bytes))
def transaction_exists(self, hash_bytes: bytes) -> bool: if hash_bytes in self.cache: return True return skip_warning(self.store.transaction_exists)(hash_bytes)
def save_transaction(self, tx: BaseTransaction, *, only_metadata: bool = False) -> None: self._save_transaction(tx) self._save_to_weakref(tx) # call super which adds to index if needed skip_warning(super().save_transaction)(tx, only_metadata=only_metadata)
def remove_transaction(self, tx: BaseTransaction) -> None: assert tx.hash is not None skip_warning(super().remove_transaction)(tx) self.cache.pop(tx.hash, None) self.dirty_txs.discard(tx.hash) self._remove_from_weakref(tx)
def remove_transaction(self, tx: 'BaseTransaction') -> None: skip_warning(super().remove_transaction)(tx) self._db.delete(tx.hash) self._remove_from_weakref(tx)
def remove_transaction_deferred(self, tx: BaseTransaction) -> Deferred: return succeed(skip_warning(self.remove_transaction)(tx))
def get_count_tx_blocks(self) -> int: self._flush_to_storage(self.dirty_txs.copy()) return skip_warning(self.store.get_count_tx_blocks)()
def get_transaction_deferred(self, hash_bytes: bytes) -> Deferred: return succeed(skip_warning(self.get_transaction)(hash_bytes))
def Exists(self, request: protos.ExistsRequest, context: _Context) -> protos.ExistsResponse: hash_bytes = request.hash exists = skip_warning(self.storage.transaction_exists)(hash_bytes) return protos.ExistsResponse(exists=exists)
def get_count_tx_blocks_deferred(self) -> Deferred: return succeed(skip_warning(self.get_count_tx_blocks)())
def remove_transaction(self, tx: BaseTransaction) -> None: assert tx.hash is not None skip_warning(super().remove_transaction)(tx) self.transactions.pop(tx.hash, None) self.metadata.pop(tx.hash, None)