class ClientTxnLog(HasFileStorage): """ An immutable log of transactions made by the client. """ def __init__(self, name, baseDir=None): self.dataDir = "data/clients" self.name = name HasFileStorage.__init__(self, name, baseDir=baseDir, dataDir=self.dataDir) self.clientDataLocation = self.dataLocation if not os.path.exists(self.clientDataLocation): os.makedirs(self.clientDataLocation) self.transactionLog = TextFileStore(self.clientDataLocation, "transactions") self.serializer = CompactSerializer(fields=self.txnFieldOrdering) def close(self): self.transactionLog.close() @property def txnFieldOrdering(self): fields = getTxnOrderedFields() return updateFieldsWithSeqNo(fields) def append(self, identifier: str, reqId, txn): key = '{}{}'.format(identifier, reqId) self.transactionLog.put(key=key, value=self.serializer.serialize( txn, fields=self.txnFieldOrdering, toBytes=False)) def hasTxn(self, identifier, reqId) -> bool: key = '{}{}'.format(identifier, reqId) for key in self.transactionLog.iterator(includeKey=True, includeValue=False): if key == str(reqId): return True return False def reset(self): self.transactionLog.reset()
class Ledger(ImmutableStore): def __init__(self, tree: MerkleTree, dataDir: str, serializer: MappingSerializer = None, fileName: str = None, ensureDurability: bool = True): """ :param tree: an implementation of MerkleTree :param dataDir: the directory where the transaction log is stored :param serializer: an object that can serialize the data before hashing it and storing it in the MerkleTree :param fileName: the name of the transaction log file """ self.dataDir = dataDir self.tree = tree self.leafSerializer = serializer or \ JsonSerializer() # type: MappingSerializer self.hasher = TreeHasher() self._transactionLog = None # type: FileStore self._transactionLogName = fileName or "transactions" self.ensureDurability = ensureDurability self.start(ensureDurability=ensureDurability) self.seqNo = 0 self.recoverTree() def recoverTree(self): # TODO: Should probably have 2 classes of hash store, # persistent and non persistent # TODO: this definitely should be done in a more generic way: if not isinstance(self.tree, CompactMerkleTree): logging.error("Do not know how to recover {}".format(self.tree)) raise TypeError("Merkle tree type {} is not supported".format( type(self.tree))) # ATTENTION! # This functionality is disabled until better consistency verification # implemented - always using recovery from transaction log # if not self.tree.hashStore \ # or isinstance(self.tree.hashStore, MemoryHashStore) \ # or self.tree.leafCount == 0: # logging.debug("Recovering tree from transaction log") # self.recoverTreeFromTxnLog() # else: # try: # logging.debug("Recovering tree from hash store of size {}". # format(self.tree.leafCount)) # self.recoverTreeFromHashStore() # except ConsistencyVerificationFailed: # logging.error("Consistency verification of merkle tree " # "from hash store failed, " # "falling back to transaction log") # self.recoverTreeFromTxnLog() logging.debug("Recovering tree from transaction log") start = time.perf_counter() self.recoverTreeFromTxnLog() end = time.perf_counter() t = end - start logging.debug( "Recovered tree from transaction log in {} seconds".format(t)) def recoverTreeFromTxnLog(self): self.tree.hashStore.reset() for key, entry in self._transactionLog.iterator(): record = self.leafSerializer.deserialize(entry) self._addToTree(record) def recoverTreeFromHashStore(self): treeSize = self.tree.leafCount self.seqNo = treeSize hashes = list( reversed(self.tree.inclusion_proof(treeSize, treeSize + 1))) self.tree._update(self.tree.leafCount, hashes) self.tree.verifyConsistency(self._transactionLog.numKeys) def add(self, leaf): self._addToStore(leaf) merkleInfo = self._addToTree(leaf) return merkleInfo def _addToTree(self, leafData): serializedLeafData = self.serializeLeaf(leafData) auditPath = self.tree.append(serializedLeafData) self.seqNo += 1 merkleInfo = { F.seqNo.name: self.seqNo, F.rootHash.name: base64.b64encode(self.tree.root_hash).decode(), F.auditPath.name: [base64.b64encode(h).decode() for h in auditPath] } return merkleInfo def _addToStore(self, data): key = str(self.seqNo + 1) self._transactionLog.put(key=key, value=self.leafSerializer.serialize( data, toBytes=False)) def append(self, txn): merkleInfo = self.add(txn) return merkleInfo def get(self, **kwargs): for seqNo, value in self._transactionLog.iterator(): data = self.leafSerializer.deserialize(value) # If `kwargs` is a subset of `data` if set(kwargs.values()) == {data.get(k) for k in kwargs.keys()}: data[F.seqNo.name] = int(seqNo) return data def getBySeqNo(self, seqNo): key = str(seqNo) value = self._transactionLog.get(key) if value: return self.leafSerializer.deserialize(value) else: return value def __getitem__(self, seqNo): return self.getBySeqNo(seqNo) def lastCount(self): key = self._transactionLog.lastKey return 0 if key is None else int(key) def serializeLeaf(self, leafData): return self.leafSerializer.serialize(leafData) @property def size(self) -> int: return self.tree.tree_size def __len__(self): return self.size @property def root_hash(self) -> str: return base64.b64encode(self.tree.root_hash).decode() def merkleInfo(self, seqNo): seqNo = int(seqNo) assert seqNo > 0 rootHash = self.tree.merkle_tree_hash(0, seqNo) auditPath = self.tree.inclusion_proof(seqNo - 1, seqNo) return { F.rootHash.name: base64.b64encode(rootHash).decode(), F.auditPath.name: [base64.b64encode(h).decode() for h in auditPath] } def start(self, loop=None, ensureDurability=True): if self._transactionLog and not self._transactionLog.closed: logging.debug("Ledger already started.") else: logging.debug("Starting ledger...") ensureDurability = ensureDurability or self.ensureDurability self._transactionLog = TextFileStore( self.dataDir, self._transactionLogName, isLineNoKey=True, storeContentHash=False, ensureDurability=ensureDurability) def stop(self): self._transactionLog.close() def reset(self): self._transactionLog.reset() def getAllTxn(self, frm: int = None, to: int = None): result = OrderedDict() for seqNo, txn in self._transactionLog.iterator(): seqNo = int(seqNo) if (frm is None or seqNo >= frm) and \ (to is None or seqNo <= to): result[seqNo] = self.leafSerializer.deserialize(txn) if to is not None and seqNo > to: break return result