def testMeasureWriteTime(tempdir):
store = TextFileStore(tempdir,
'benchWithSync',
isLineNoKey=True,
storeContentHash=False)
hashes = [hexlify(h).decode() for h in generateHashes(1000)]
start = time.time()
for h in hashes:
store.put(value=h)
timeTakenWithSync = time.time() - start
store = TextFileStore(tempdir,
'benchWithoutSync',
isLineNoKey=True,
storeContentHash=False,
ensureDurability=False)
start = time.time()
for h in hashes:
store.put(value=h)
timeTakenWithoutSync = time.time() - start
print("Time taken to write {} entries to file with fsync is {} "
"seconds".format(len(hashes), timeTakenWithSync))
print("Time taken to write {} entries to file without fsync is {} "
"seconds".format(len(hashes), timeTakenWithoutSync))
print("So the difference is {} seconds".format(timeTakenWithSync -
timeTakenWithoutSync))
# On most platforms the ratio between write time with fsync and
# write time without fsync typically must be greater than 100.
# But on Windows Server 2012 this ratio may be less - down to 30.
assert timeTakenWithoutSync*10 < timeTakenWithSync, "ratio is {}".\
format(timeTakenWithSync/timeTakenWithoutSync)
def testMeasureWriteTime(tempdir):
store = TextFileStore(tempdir, 'benchWithSync', isLineNoKey=True,
storeContentHash=False)
hashes = [hexlify(h).decode() for h in generateHashes(1000)]
start = time.time()
for h in hashes:
store.put(value=h)
timeTakenWithSync = time.time() - start
store = TextFileStore(tempdir, 'benchWithoutSync', isLineNoKey=True,
storeContentHash=False, ensureDurability=False)
start = time.time()
for h in hashes:
store.put(value=h)
timeTakenWithoutSync = time.time() - start
print("Time taken to write {} entries to file with fsync is {} "
"seconds".format(len(hashes), timeTakenWithSync))
print("Time taken to write {} entries to file without fsync is {} "
"seconds".format(len(hashes), timeTakenWithoutSync))
print("So the difference is {} seconds".
format(timeTakenWithSync-timeTakenWithoutSync))
# On most platforms the ratio between write time with fsync and
# write time without fsync typically must be greater than 100.
# But on Windows Server 2012 this ratio may be less - down to 30.
assert timeTakenWithoutSync*10 < timeTakenWithSync, "ratio is {}".\
format(timeTakenWithSync/timeTakenWithoutSync)
class ClientTxnLog(HasFileStorage):
"""
An immutable log of transactions made by the client.
"""
def __init__(self, name, baseDir=None):
self.dataDir = "data/clients"
self.name = name
HasFileStorage.__init__(self, name,
baseDir=baseDir,
dataDir=self.dataDir)
self.clientDataLocation = self.dataLocation
if not os.path.exists(self.clientDataLocation):
os.makedirs(self.clientDataLocation)
self.transactionLog = TextFileStore(self.clientDataLocation,
"transactions")
self.serializer = CompactSerializer(fields=self.txnFieldOrdering)
@property
def txnFieldOrdering(self):
fields = getTxnOrderedFields()
return updateFieldsWithSeqNo(fields)
def append(self, reqId, txn):
self.transactionLog.put(key=str(reqId), value=self.serializer.serialize(txn,
fields=self.txnFieldOrdering, toBytes=False))
def hasTxnWithReqId(self, reqId) -> bool:
for key in self.transactionLog.iterator(includeKey=True,
includeValue=False):
if key == str(reqId):
return True
return False
class ClientTxnLog(HasFileStorage):
"""
An immutable log of transactions made by the client.
"""
def __init__(self, name, baseDir=None):
self.dataDir = "data/clients"
self.name = name
HasFileStorage.__init__(self, name,
baseDir=baseDir,
dataDir=self.dataDir)
self.clientDataLocation = self.dataLocation
if not os.path.exists(self.clientDataLocation):
os.makedirs(self.clientDataLocation)
self.transactionLog = TextFileStore(self.clientDataLocation,
"transactions")
self.serializer = CompactSerializer(fields=self.txnFieldOrdering)
@property
def txnFieldOrdering(self):
fields = getTxnOrderedFields()
return updateFieldsWithSeqNo(fields)
def append(self, identifier: str, reqId, txn):
key = '{}{}'.format(identifier, reqId)
self.transactionLog.put(key=key, value=self.serializer.serialize(txn,
fields=self.txnFieldOrdering, toBytes=False))
def hasTxn(self, identifier, reqId) -> bool:
key = '{}{}'.format(identifier, reqId)
for key in self.transactionLog.iterator(includeKey=True,
includeValue=False):
if key == str(reqId):
return True
return False
class EntityFileStore(EntityStore):
def __init__(self, name: str, dataDir: str):
self._db = TextFileStore(dbName=name, dbDir=dataDir)
def add(self, name: str, entity):
self._db.put(name, entity)
def get(self, name: str):
return self._db.get(name)
def test_equality_to_text_file_store(tmpdir):
"""
This test verifies that TextFileStore and ChunkedFileStore behave equally
"""
isLineNoKey = True
storeContentHash = False
ensureDurability = True
dbDir = str(tmpdir)
defaultFile = os.path.join(dbDir, "template")
lines = [
"FirstLine\n", "OneMoreLine\n", "AnotherLine\n", "LastDefaultLine\n"
]
with open(defaultFile, "w") as f:
f.writelines(lines)
chunkSize = len(lines)
chunkedStore = ChunkedFileStore(dbDir=dbDir,
dbName="chunked_data",
isLineNoKey=isLineNoKey,
storeContentHash=storeContentHash,
chunkSize=chunkSize,
ensureDurability=ensureDurability,
chunkStoreConstructor=TextFileStore,
defaultFile=defaultFile)
textStore = TextFileStore(dbDir=dbDir,
dbName="text_data",
isLineNoKey=isLineNoKey,
storeContentHash=storeContentHash,
ensureDurability=ensureDurability,
defaultFile=defaultFile)
for i in range(1, 5 * chunkSize):
value = str(i)
chunkedStore.put(value)
textStore.put(value)
assert textStore.get(value) == chunkedStore.get(value)
assert list(chunkedStore.iterator()) == \
list(textStore.iterator())
