def testMeasureWriteTime(tempdir):
store = TextFileStore(tempdir,
'benchWithSync',
isLineNoKey=True,
storeContentHash=False)
hashes = [hexlify(h).decode() for h in generateHashes(1000)]
start = time.time()
for h in hashes:
store.put(value=h)
timeTakenWithSync = time.time() - start
store = TextFileStore(tempdir,
'benchWithoutSync',
isLineNoKey=True,
storeContentHash=False,
ensureDurability=False)
start = time.time()
for h in hashes:
store.put(value=h)
timeTakenWithoutSync = time.time() - start
print("Time taken to write {} entries to file with fsync is {} "
"seconds".format(len(hashes), timeTakenWithSync))
print("Time taken to write {} entries to file without fsync is {} "
"seconds".format(len(hashes), timeTakenWithoutSync))
print("So the difference is {} seconds".format(timeTakenWithSync -
timeTakenWithoutSync))
# On most platforms the ratio between write time with fsync and
# write time without fsync typically must be greater than 100.
# But on Windows Server 2012 this ratio may be less - down to 30.
assert timeTakenWithoutSync*10 < timeTakenWithSync, "ratio is {}".\
format(timeTakenWithSync/timeTakenWithoutSync)
def _defaultStore(dataDir, logName, ensureDurability,
defaultFile) -> FileStore:
return TextFileStore(dataDir,
logName,
isLineNoKey=True,
storeContentHash=False,
ensureDurability=ensureDurability,
defaultFile=defaultFile)
def __init__(self, name, baseDir=None):
self.dataDir = "data/clients"
self.name = name
HasFileStorage.__init__(self, name,
baseDir=baseDir,
dataDir=self.dataDir)
self.clientDataLocation = self.dataLocation
if not os.path.exists(self.clientDataLocation):
os.makedirs(self.clientDataLocation)
self.transactionLog = TextFileStore(self.clientDataLocation,
"transactions")
self.serializer = CompactSerializer(fields=self.txnFieldOrdering)
def start(self, loop=None, ensureDurability=True):
if self._transactionLog and not self._transactionLog.closed:
logging.debug("Ledger already started.")
else:
logging.debug("Starting ledger...")
ensureDurability = ensureDurability or self.ensureDurability
self._transactionLog = TextFileStore(
self.dataDir,
self._transactionLogName,
isLineNoKey=True,
storeContentHash=False,
ensureDurability=ensureDurability)
def initStorage(storageType, name, dataDir=None, config=None):
if storageType == StorageType.File:
if dataDir is None:
raise DataDirectoryNotFound
return TextFileStore(dataDir, name)
elif storageType == StorageType.OrientDB:
if config is None:
raise DBConfigNotFound
orientConf = config.OrientDB
return OrientDbStore(user=orientConf["user"],
password=orientConf["password"],
host=orientConf["host"],
port=orientConf["port"],
dbName=name)
def test_equality_to_text_file_store(tmpdir):
"""
This test verifies that TextFileStore and ChunkedFileStore behave equally
"""
isLineNoKey = True
storeContentHash = False
ensureDurability = True
dbDir = str(tmpdir)
defaultFile = os.path.join(dbDir, "template")
lines = [
"FirstLine\n", "OneMoreLine\n", "AnotherLine\n", "LastDefaultLine\n"
]
with open(defaultFile, "w") as f:
f.writelines(lines)
chunkSize = len(lines)
chunkedStore = ChunkedFileStore(dbDir=dbDir,
dbName="chunked_data",
isLineNoKey=isLineNoKey,
storeContentHash=storeContentHash,
chunkSize=chunkSize,
ensureDurability=ensureDurability,
chunkStoreConstructor=TextFileStore,
defaultFile=defaultFile)
textStore = TextFileStore(dbDir=dbDir,
dbName="text_data",
isLineNoKey=isLineNoKey,
storeContentHash=storeContentHash,
ensureDurability=ensureDurability,
defaultFile=defaultFile)
for i in range(1, 5 * chunkSize):
value = str(i)
chunkedStore.put(value)
textStore.put(value)
assert textStore.get(value) == chunkedStore.get(value)
assert list(chunkedStore.iterator()) == \
list(textStore.iterator())
def __init__(self, name: str, dataDir: str):
self._db = TextFileStore(dbName=name, dbDir=dataDir)
def getLedger(baseDir, dbName, storeHash=True, isLineNoKey: bool = False):
return TextFileStore(dbDir=baseDir,
dbName=dbName,
storeContentHash=storeHash,
isLineNoKey=isLineNoKey)
def initStorage(storageType, name, dataDir=None, config=None):
if storageType == StorageType.File:
if dataDir is None:
raise DataDirectoryNotFound
return TextFileStore(dataDir, name)
