import pysos
import time
N = 100 * 1000
db = pysos.Dict("temp/test.db")
t = time.time()
for i in range(N):
db["key_" + str(i)] = {"some": "object_" + str(i)}
dt = time.time() - t
print(f'Writes: {int(N / dt)} / second')
t = time.time()
for i in range(N):
value = db["key_" + str(i)]
dt = time.time() - t
print(f'Reads: {int(N / dt)} / second')
db.close()
help='enable (increasingly verbose) debug diagnostics',
action='count')
if sys.argv[0].find('pydoc') > 0:
sys.argv = [sys.argv[0]]
args = parser.parse_args()
# set loglevel to use (DEBUG/INFO/WARNING) and message format
loglevel = logging.INFO
if args.verbose != None and args.verbose > 0: loglevel = logging.DEBUG
logging.basicConfig(level=loglevel,
format='%(asctime)s %(name)s %(levelname)s: %(message)s',
datefmt='%d/%m/%Y %H:%M:%S')
# create 2 DHT server nodes
host1, port1 = 'localhost', 14900
if have_storage: store1 = pysos.Dict('sto%d.dat' % port1)
else: store1 = {}
dht1 = DHT(host1, port1, storage=store1)
myseeds = [(host1, port1)]
host2, port2 = 'localhost', 14901
if have_storage: store2 = pysos.Dict('sto%d.dat' % port2)
else: store2 = {}
dht2 = DHT(host2, port2, seeds=myseeds, storage=store2)
# store some values via DHT1 or 2
dht1["my_key"] = [u"My", u"json-serializable", u"Object"]
dht1["akey1"] = ["Some", "Strings", "for key1"]
dht1["akey2"] = ["Other", "Strings", "for key2"]
dht2["akey3"] = ["Further", "Strings", "for key3"]
# demo retrieve data from nodes
def run_bench(N, db_tpl):
batchsize = 1000
LMDBM_FILE = db_tpl.format("lmdbm")
PYSOS_FILE = db_tpl.format("pysos")
SQLITEDICT_FILE = db_tpl.format("sqlitedict")
DBM_FILE = db_tpl.format("dbm")
remove_lmdbm(LMDBM_FILE)
with suppress(FileNotFoundError):
os.unlink(PYSOS_FILE)
with suppress(FileNotFoundError):
os.unlink(SQLITEDICT_FILE)
remove_dbm(DBM_FILE)
ret = defaultdict(dict)
# writes
""" # without batch
with PrintStatementTime("lmdbm (no batch) {} writes: {{delta:.02f}}".format(N)):
db = JsonLmdb.open(LMDBM_FILE, "c")
for k, v in data(N):
db[k] = v
db.close()
remove_lmdbm(LMDBM_FILE)
"""
with MeasureTime() as t:
with JsonLmdb.open(LMDBM_FILE, "c") as db:
for pairs in batch(data(N), batchsize):
db.update(pairs)
ret["lmdbm"]["write"] = t.get()
print("lmdbm batch write", N, t.get())
with open(os.devnull, "w") as devnull: # mute annoying "free lines" output
with redirect_stdout(devnull):
with MeasureTime() as t:
db = pysos.Dict(PYSOS_FILE)
for k, v in data(N):
db[k] = v
db.close()
ret["pysos"]["write"] = t.get()
print("pysos write", N, t.get())
with MeasureTime() as t:
with SqliteDict(SQLITEDICT_FILE) as db:
for pairs in batch(data(N), batchsize):
db.update(pairs)
db.commit()
ret["sqlitedict"]["write"] = t.get()
print("sqlitedict batch write", N, t.get())
with MeasureTime() as t:
with dbm.open(DBM_FILE, "c") as db:
for k, v in data(N):
db[k] = json.dumps(v)
ret["dbm"]["write"] = t.get()
print("dbm write", N, t.get())
# reads
with MeasureTime() as t:
with JsonLmdb.open(LMDBM_FILE, "r") as db:
for k in allkeys(N):
db[k]
#ret["lmdbm"]["read"] = t.get()
print("lmdbm cont read", N, t.get())
with MeasureTime() as t:
with JsonLmdb.open(LMDBM_FILE, "r") as db:
for k in randkeys(N, N):
db[k]
ret["lmdbm"]["read"] = t.get()
print("lmdbm rand read", N, t.get())
with open(os.devnull, "w") as devnull: # mute annoying "free lines" output
with redirect_stdout(devnull):
with MeasureTime() as t:
db = pysos.Dict(PYSOS_FILE)
for k in randkeys(N, N):
db[k]
db.close()
ret["pysos"]["read"] = t.get()
print("pysos read", N, t.get())
with MeasureTime() as t:
with SqliteDict(SQLITEDICT_FILE) as db:
for k in randkeys(N, N):
db[k]
ret["sqlitedict"]["read"] = t.get()
print("sqlitedict read", N, t.get())
with MeasureTime() as t:
with dbm.open(DBM_FILE, "r") as db:
for k in randkeys(N, N):
json.loads(db[k])
ret["dbm"]["read"] = t.get()
print("dbm read", N, t.get())
return ret
def run_bench(N, db_tpl) -> Dict[str, Dict[str, float]]:
batchsize = 1000
LMDBM_FILE = db_tpl.format("lmdbm")
LMDBM_BATCH_FILE = db_tpl.format("lmdbm-batch")
PYSOS_FILE = db_tpl.format("pysos")
SQLITEDICT_FILE = db_tpl.format("sqlitedict")
SQLITEDICT_BATCH_FILE = db_tpl.format("sqlitedict-batch")
DBM_DUMB_FILE = db_tpl.format("dbm.dumb")
DBM_GNU_FILE = db_tpl.format("dbm.gnu")
SEMIDBM_FILE = db_tpl.format("semidbm")
VEDIS_FILE = db_tpl.format("vedis")
VEDIS_BATCH_FILE = db_tpl.format("vedis-batch")
UNQLITE_FILE = db_tpl.format("unqlite")
UNQLITE_BATCH_FILE = db_tpl.format("unqlite-batch")
remove_lmdbm(LMDBM_FILE)
remove_lmdbm(LMDBM_BATCH_FILE)
with suppress(FileNotFoundError):
os.unlink(PYSOS_FILE)
with suppress(FileNotFoundError):
os.unlink(SQLITEDICT_FILE)
with suppress(FileNotFoundError):
os.unlink(SQLITEDICT_BATCH_FILE)
remove_dbm(DBM_DUMB_FILE)
remove_semidbm(SEMIDBM_FILE)
with suppress(FileNotFoundError):
os.unlink(VEDIS_FILE)
with suppress(FileNotFoundError):
os.unlink(VEDIS_BATCH_FILE)
with suppress(FileNotFoundError):
os.unlink(UNQLITE_FILE)
with suppress(FileNotFoundError):
os.unlink(UNQLITE_BATCH_FILE)
ret: DefaultDict[str, Dict[str, float]] = defaultdict(dict)
# writes
with MeasureTime() as t:
with JsonLmdb.open(LMDBM_FILE, "c") as db:
for k, v in data(N):
db[k] = v
ret["lmdbm"]["write"] = t.get()
print("lmdbm write", N, t.get())
with MeasureTime() as t:
with JsonLmdb.open(LMDBM_BATCH_FILE, "c") as db:
for pairs in batch(data(N), batchsize):
db.update(pairs)
ret["lmdbm-batch"]["write"] = t.get()
print("lmdbm-batch write", N, t.get())
with open(os.devnull, "w") as devnull: # mute annoying "free lines" output
with redirect_stdout(devnull):
with MeasureTime() as t:
db = pysos.Dict(PYSOS_FILE)
for k, v in data(N):
db[k] = v
db.close()
ret["pysos"]["write"] = t.get()
print("pysos write", N, t.get())
with MeasureTime() as t:
with SqliteDict(SQLITEDICT_FILE, autocommit=True) as db:
for k, v in data(N):
db[k] = v
ret["sqlitedict"]["write"] = t.get()
print("sqlitedict write", N, t.get())
with MeasureTime() as t:
with SqliteDict(SQLITEDICT_BATCH_FILE, autocommit=False) as db:
for pairs in batch(data(N), batchsize):
db.update(pairs)
db.commit()
ret["sqlitedict-batch"]["write"] = t.get()
print("sqlitedict-batch write", N, t.get())
with MeasureTime() as t:
with dbm.dumb.open(DBM_DUMB_FILE, "c") as db:
for k, v in data(N):
db[k] = json.dumps(v)
ret["dbm.dumb"]["write"] = t.get()
print("dbm.dumb write", N, t.get())
if gdbm:
with MeasureTime() as t:
with dbm.gnu.open(DBM_GNU_FILE, "c") as db:
for k, v in data(N):
db[k] = json.dumps(v)
ret["dbm.gnu"]["write"] = t.get()
print("dbm.gnu write", N, t.get())
with MeasureTime() as t:
db = semidbm.open(SEMIDBM_FILE, "c")
for k, v in data(N):
db[k] = json.dumps(v)
db.close()
ret["semidbm"]["write"] = t.get()
print("semidbm write", N, t.get())
with MeasureTime() as t:
with Vedis(VEDIS_FILE) as db:
for k, v in data(N):
db[k] = json.dumps(v)
ret["vedis"]["write"] = t.get()
print("vedis write", N, t.get())
with MeasureTime() as t:
with Vedis(VEDIS_BATCH_FILE) as db:
for pairs in batch(data(N), batchsize):
db.update({k: json.dumps(v) for k, v in pairs})
ret["vedis-batch"]["write"] = t.get()
print("vedis-batch write", N, t.get())
with MeasureTime() as t:
with UnQLite(UNQLITE_FILE) as db:
for k, v in data(N):
db[k] = json.dumps(v)
ret["unqlite"]["write"] = t.get()
print("unqlite write", N, t.get())
with MeasureTime() as t:
with UnQLite(UNQLITE_BATCH_FILE) as db:
for pairs in batch(data(N), batchsize):
db.update({k: json.dumps(v) for k, v in pairs})
ret["unqlite-batch"]["write"] = t.get()
print("unqlite-batch write", N, t.get())
# reads
with MeasureTime() as t:
with JsonLmdb.open(LMDBM_FILE, "r") as db:
for k in allkeys(N):
db[k]
# ret["lmdbm"]["read"] = t.get()
print("lmdbm cont read", N, t.get())
with MeasureTime() as t:
with JsonLmdb.open(LMDBM_FILE, "r") as db:
for k in randkeys(N, N):
db[k]
ret["lmdbm"]["read"] = t.get()
print("lmdbm rand read", N, t.get())
with open(os.devnull, "w") as devnull: # mute annoying "free lines" output
with redirect_stdout(devnull):
with MeasureTime() as t:
db = pysos.Dict(PYSOS_FILE)
for k in randkeys(N, N):
db[k]
db.close()
ret["pysos"]["read"] = t.get()
print("pysos read", N, t.get())
with MeasureTime() as t:
with SqliteDict(SQLITEDICT_FILE) as db:
for k in randkeys(N, N):
db[k]
ret["sqlitedict"]["read"] = t.get()
print("sqlitedict read", N, t.get())
with MeasureTime() as t:
with dbm.dumb.open(DBM_DUMB_FILE, "r") as db:
for k in randkeys(N, N):
json.loads(db[k])
ret["dbm.dumb"]["read"] = t.get()
print("dbm.dumb read", N, t.get())
if gdbm:
with MeasureTime() as t:
with dbm.gnu.open(DBM_GNU_FILE, "r") as db:
for k in randkeys(N, N):
json.loads(db[k])
ret["dbm.gnu"]["read"] = t.get()
print("dbm.gnu read", N, t.get())
with MeasureTime() as t:
db = semidbm.open(SEMIDBM_FILE, "r")
for k in randkeys(N, N):
json.loads(db[k])
db.close()
ret["semidbm"]["read"] = t.get()
print("semidbm read", N, t.get())
with MeasureTime() as t:
with Vedis(VEDIS_FILE) as db:
for k in randkeys(N, N):
json.loads(db[k])
ret["vedis"]["read"] = t.get()
print("vedis read", N, t.get())
with MeasureTime() as t:
with UnQLite(UNQLITE_FILE) as db:
for k in randkeys(N, N):
json.loads(db[k])
ret["unqlite"]["read"] = t.get()
print("unqlite read", N, t.get())
return ret
import time
import test_rnd as rnd
import random
import pysos
# initialize the data
N = 1234
items = [(rnd.utf8(20), rnd.utf8(200)) for i in range(N)]
start = time.time()
db = pysos.Dict('temp/sos_dict')
#import shelve
#db = shelve.open('temp.shelve')
print("%.2fs: %d items loaded" % (time.time() - start, len(db)))
# add all items
for key,val in items:
db[key] = val
print("%.2fs: %d items added" % (time.time() - start, len(items)))
# read all keys
random.shuffle(items)
for key,val in items:
val2 = db[key]
assert val2 == val
print("%.2fs: %d items read" % (time.time() - start, len(items)))