def testMeasureWriteTime(tempdir): store = TextFileStore(tempdir, 'benchWithSync', isLineNoKey=True, storeContentHash=False) hashes = [hexlify(h).decode() for h in generateHashes(1000)] start = time.time() for h in hashes: store.put(value=h) timeTakenWithSync = time.time() - start store = TextFileStore(tempdir, 'benchWithoutSync', isLineNoKey=True, storeContentHash=False, ensureDurability=False) start = time.time() for h in hashes: store.put(value=h) timeTakenWithoutSync = time.time() - start print("Time taken to write {} entries to file with fsync is {} " "seconds".format(len(hashes), timeTakenWithSync)) print("Time taken to write {} entries to file without fsync is {} " "seconds".format(len(hashes), timeTakenWithoutSync)) print("So the difference is {} seconds".format(timeTakenWithSync - timeTakenWithoutSync)) # On most platforms the ratio between write time with fsync and # write time without fsync typically must be greater than 100. # But on Windows Server 2012 this ratio may be less - down to 30. assert timeTakenWithoutSync*10 < timeTakenWithSync, "ratio is {}".\ format(timeTakenWithSync/timeTakenWithoutSync)
def testMeasureWriteTime(tempdir): store = TextFileStore(tempdir, 'benchWithSync', isLineNoKey=True, storeContentHash=False) hashes = [hexlify(h).decode() for h in generateHashes(1000)] start = time.time() for h in hashes: store.put(value=h) timeTakenWithSync = time.time() - start store = TextFileStore(tempdir, 'benchWithoutSync', isLineNoKey=True, storeContentHash=False, ensureDurability=False) start = time.time() for h in hashes: store.put(value=h) timeTakenWithoutSync = time.time() - start print("Time taken to write {} entries to file with fsync is {} " "seconds".format(len(hashes), timeTakenWithSync)) print("Time taken to write {} entries to file without fsync is {} " "seconds".format(len(hashes), timeTakenWithoutSync)) print("So the difference is {} seconds". format(timeTakenWithSync-timeTakenWithoutSync)) # On most platforms the ratio between write time with fsync and # write time without fsync typically must be greater than 100. # But on Windows Server 2012 this ratio may be less - down to 30. assert timeTakenWithoutSync*10 < timeTakenWithSync, "ratio is {}".\ format(timeTakenWithSync/timeTakenWithoutSync)
class ClientTxnLog(HasFileStorage): """ An immutable log of transactions made by the client. """ def __init__(self, name, baseDir=None): self.dataDir = "data/clients" self.name = name HasFileStorage.__init__(self, name, baseDir=baseDir, dataDir=self.dataDir) self.clientDataLocation = self.dataLocation if not os.path.exists(self.clientDataLocation): os.makedirs(self.clientDataLocation) self.transactionLog = TextFileStore(self.clientDataLocation, "transactions") self.serializer = CompactSerializer(fields=self.txnFieldOrdering) @property def txnFieldOrdering(self): fields = getTxnOrderedFields() return updateFieldsWithSeqNo(fields) def append(self, reqId, txn): self.transactionLog.put(key=str(reqId), value=self.serializer.serialize(txn, fields=self.txnFieldOrdering, toBytes=False)) def hasTxnWithReqId(self, reqId) -> bool: for key in self.transactionLog.iterator(includeKey=True, includeValue=False): if key == str(reqId): return True return False
class ClientTxnLog(HasFileStorage): """ An immutable log of transactions made by the client. """ def __init__(self, name, baseDir=None): self.dataDir = "data/clients" self.name = name HasFileStorage.__init__(self, name, baseDir=baseDir, dataDir=self.dataDir) self.clientDataLocation = self.dataLocation if not os.path.exists(self.clientDataLocation): os.makedirs(self.clientDataLocation) self.transactionLog = TextFileStore(self.clientDataLocation, "transactions") self.serializer = CompactSerializer(fields=self.txnFieldOrdering) @property def txnFieldOrdering(self): fields = getTxnOrderedFields() return updateFieldsWithSeqNo(fields) def append(self, identifier: str, reqId, txn): key = '{}{}'.format(identifier, reqId) self.transactionLog.put(key=key, value=self.serializer.serialize(txn, fields=self.txnFieldOrdering, toBytes=False)) def hasTxn(self, identifier, reqId) -> bool: key = '{}{}'.format(identifier, reqId) for key in self.transactionLog.iterator(includeKey=True, includeValue=False): if key == str(reqId): return True return False
class EntityFileStore(EntityStore): def __init__(self, name: str, dataDir: str): self._db = TextFileStore(dbName=name, dbDir=dataDir) def add(self, name: str, entity): self._db.put(name, entity) def get(self, name: str): return self._db.get(name)
def test_equality_to_text_file_store(tmpdir): """ This test verifies that TextFileStore and ChunkedFileStore behave equally """ isLineNoKey = True storeContentHash = False ensureDurability = True dbDir = str(tmpdir) defaultFile = os.path.join(dbDir, "template") lines = [ "FirstLine\n", "OneMoreLine\n", "AnotherLine\n", "LastDefaultLine\n" ] with open(defaultFile, "w") as f: f.writelines(lines) chunkSize = len(lines) chunkedStore = ChunkedFileStore(dbDir=dbDir, dbName="chunked_data", isLineNoKey=isLineNoKey, storeContentHash=storeContentHash, chunkSize=chunkSize, ensureDurability=ensureDurability, chunkStoreConstructor=TextFileStore, defaultFile=defaultFile) textStore = TextFileStore(dbDir=dbDir, dbName="text_data", isLineNoKey=isLineNoKey, storeContentHash=storeContentHash, ensureDurability=ensureDurability, defaultFile=defaultFile) for i in range(1, 5 * chunkSize): value = str(i) chunkedStore.put(value) textStore.put(value) assert textStore.get(value) == chunkedStore.get(value) assert list(chunkedStore.iterator()) == \ list(textStore.iterator())