import pysos import time N = 100 * 1000 db = pysos.Dict("temp/test.db") t = time.time() for i in range(N): db["key_" + str(i)] = {"some": "object_" + str(i)} dt = time.time() - t print(f'Writes: {int(N / dt)} / second') t = time.time() for i in range(N): value = db["key_" + str(i)] dt = time.time() - t print(f'Reads: {int(N / dt)} / second') db.close()
help='enable (increasingly verbose) debug diagnostics', action='count') if sys.argv[0].find('pydoc') > 0: sys.argv = [sys.argv[0]] args = parser.parse_args() # set loglevel to use (DEBUG/INFO/WARNING) and message format loglevel = logging.INFO if args.verbose != None and args.verbose > 0: loglevel = logging.DEBUG logging.basicConfig(level=loglevel, format='%(asctime)s %(name)s %(levelname)s: %(message)s', datefmt='%d/%m/%Y %H:%M:%S') # create 2 DHT server nodes host1, port1 = 'localhost', 14900 if have_storage: store1 = pysos.Dict('sto%d.dat' % port1) else: store1 = {} dht1 = DHT(host1, port1, storage=store1) myseeds = [(host1, port1)] host2, port2 = 'localhost', 14901 if have_storage: store2 = pysos.Dict('sto%d.dat' % port2) else: store2 = {} dht2 = DHT(host2, port2, seeds=myseeds, storage=store2) # store some values via DHT1 or 2 dht1["my_key"] = [u"My", u"json-serializable", u"Object"] dht1["akey1"] = ["Some", "Strings", "for key1"] dht1["akey2"] = ["Other", "Strings", "for key2"] dht2["akey3"] = ["Further", "Strings", "for key3"] # demo retrieve data from nodes
def run_bench(N, db_tpl): batchsize = 1000 LMDBM_FILE = db_tpl.format("lmdbm") PYSOS_FILE = db_tpl.format("pysos") SQLITEDICT_FILE = db_tpl.format("sqlitedict") DBM_FILE = db_tpl.format("dbm") remove_lmdbm(LMDBM_FILE) with suppress(FileNotFoundError): os.unlink(PYSOS_FILE) with suppress(FileNotFoundError): os.unlink(SQLITEDICT_FILE) remove_dbm(DBM_FILE) ret = defaultdict(dict) # writes """ # without batch with PrintStatementTime("lmdbm (no batch) {} writes: {{delta:.02f}}".format(N)): db = JsonLmdb.open(LMDBM_FILE, "c") for k, v in data(N): db[k] = v db.close() remove_lmdbm(LMDBM_FILE) """ with MeasureTime() as t: with JsonLmdb.open(LMDBM_FILE, "c") as db: for pairs in batch(data(N), batchsize): db.update(pairs) ret["lmdbm"]["write"] = t.get() print("lmdbm batch write", N, t.get()) with open(os.devnull, "w") as devnull: # mute annoying "free lines" output with redirect_stdout(devnull): with MeasureTime() as t: db = pysos.Dict(PYSOS_FILE) for k, v in data(N): db[k] = v db.close() ret["pysos"]["write"] = t.get() print("pysos write", N, t.get()) with MeasureTime() as t: with SqliteDict(SQLITEDICT_FILE) as db: for pairs in batch(data(N), batchsize): db.update(pairs) db.commit() ret["sqlitedict"]["write"] = t.get() print("sqlitedict batch write", N, t.get()) with MeasureTime() as t: with dbm.open(DBM_FILE, "c") as db: for k, v in data(N): db[k] = json.dumps(v) ret["dbm"]["write"] = t.get() print("dbm write", N, t.get()) # reads with MeasureTime() as t: with JsonLmdb.open(LMDBM_FILE, "r") as db: for k in allkeys(N): db[k] #ret["lmdbm"]["read"] = t.get() print("lmdbm cont read", N, t.get()) with MeasureTime() as t: with JsonLmdb.open(LMDBM_FILE, "r") as db: for k in randkeys(N, N): db[k] ret["lmdbm"]["read"] = t.get() print("lmdbm rand read", N, t.get()) with open(os.devnull, "w") as devnull: # mute annoying "free lines" output with redirect_stdout(devnull): with MeasureTime() as t: db = pysos.Dict(PYSOS_FILE) for k in randkeys(N, N): db[k] db.close() ret["pysos"]["read"] = t.get() print("pysos read", N, t.get()) with MeasureTime() as t: with SqliteDict(SQLITEDICT_FILE) as db: for k in randkeys(N, N): db[k] ret["sqlitedict"]["read"] = t.get() print("sqlitedict read", N, t.get()) with MeasureTime() as t: with dbm.open(DBM_FILE, "r") as db: for k in randkeys(N, N): json.loads(db[k]) ret["dbm"]["read"] = t.get() print("dbm read", N, t.get()) return ret
def run_bench(N, db_tpl) -> Dict[str, Dict[str, float]]: batchsize = 1000 LMDBM_FILE = db_tpl.format("lmdbm") LMDBM_BATCH_FILE = db_tpl.format("lmdbm-batch") PYSOS_FILE = db_tpl.format("pysos") SQLITEDICT_FILE = db_tpl.format("sqlitedict") SQLITEDICT_BATCH_FILE = db_tpl.format("sqlitedict-batch") DBM_DUMB_FILE = db_tpl.format("dbm.dumb") DBM_GNU_FILE = db_tpl.format("dbm.gnu") SEMIDBM_FILE = db_tpl.format("semidbm") VEDIS_FILE = db_tpl.format("vedis") VEDIS_BATCH_FILE = db_tpl.format("vedis-batch") UNQLITE_FILE = db_tpl.format("unqlite") UNQLITE_BATCH_FILE = db_tpl.format("unqlite-batch") remove_lmdbm(LMDBM_FILE) remove_lmdbm(LMDBM_BATCH_FILE) with suppress(FileNotFoundError): os.unlink(PYSOS_FILE) with suppress(FileNotFoundError): os.unlink(SQLITEDICT_FILE) with suppress(FileNotFoundError): os.unlink(SQLITEDICT_BATCH_FILE) remove_dbm(DBM_DUMB_FILE) remove_semidbm(SEMIDBM_FILE) with suppress(FileNotFoundError): os.unlink(VEDIS_FILE) with suppress(FileNotFoundError): os.unlink(VEDIS_BATCH_FILE) with suppress(FileNotFoundError): os.unlink(UNQLITE_FILE) with suppress(FileNotFoundError): os.unlink(UNQLITE_BATCH_FILE) ret: DefaultDict[str, Dict[str, float]] = defaultdict(dict) # writes with MeasureTime() as t: with JsonLmdb.open(LMDBM_FILE, "c") as db: for k, v in data(N): db[k] = v ret["lmdbm"]["write"] = t.get() print("lmdbm write", N, t.get()) with MeasureTime() as t: with JsonLmdb.open(LMDBM_BATCH_FILE, "c") as db: for pairs in batch(data(N), batchsize): db.update(pairs) ret["lmdbm-batch"]["write"] = t.get() print("lmdbm-batch write", N, t.get()) with open(os.devnull, "w") as devnull: # mute annoying "free lines" output with redirect_stdout(devnull): with MeasureTime() as t: db = pysos.Dict(PYSOS_FILE) for k, v in data(N): db[k] = v db.close() ret["pysos"]["write"] = t.get() print("pysos write", N, t.get()) with MeasureTime() as t: with SqliteDict(SQLITEDICT_FILE, autocommit=True) as db: for k, v in data(N): db[k] = v ret["sqlitedict"]["write"] = t.get() print("sqlitedict write", N, t.get()) with MeasureTime() as t: with SqliteDict(SQLITEDICT_BATCH_FILE, autocommit=False) as db: for pairs in batch(data(N), batchsize): db.update(pairs) db.commit() ret["sqlitedict-batch"]["write"] = t.get() print("sqlitedict-batch write", N, t.get()) with MeasureTime() as t: with dbm.dumb.open(DBM_DUMB_FILE, "c") as db: for k, v in data(N): db[k] = json.dumps(v) ret["dbm.dumb"]["write"] = t.get() print("dbm.dumb write", N, t.get()) if gdbm: with MeasureTime() as t: with dbm.gnu.open(DBM_GNU_FILE, "c") as db: for k, v in data(N): db[k] = json.dumps(v) ret["dbm.gnu"]["write"] = t.get() print("dbm.gnu write", N, t.get()) with MeasureTime() as t: db = semidbm.open(SEMIDBM_FILE, "c") for k, v in data(N): db[k] = json.dumps(v) db.close() ret["semidbm"]["write"] = t.get() print("semidbm write", N, t.get()) with MeasureTime() as t: with Vedis(VEDIS_FILE) as db: for k, v in data(N): db[k] = json.dumps(v) ret["vedis"]["write"] = t.get() print("vedis write", N, t.get()) with MeasureTime() as t: with Vedis(VEDIS_BATCH_FILE) as db: for pairs in batch(data(N), batchsize): db.update({k: json.dumps(v) for k, v in pairs}) ret["vedis-batch"]["write"] = t.get() print("vedis-batch write", N, t.get()) with MeasureTime() as t: with UnQLite(UNQLITE_FILE) as db: for k, v in data(N): db[k] = json.dumps(v) ret["unqlite"]["write"] = t.get() print("unqlite write", N, t.get()) with MeasureTime() as t: with UnQLite(UNQLITE_BATCH_FILE) as db: for pairs in batch(data(N), batchsize): db.update({k: json.dumps(v) for k, v in pairs}) ret["unqlite-batch"]["write"] = t.get() print("unqlite-batch write", N, t.get()) # reads with MeasureTime() as t: with JsonLmdb.open(LMDBM_FILE, "r") as db: for k in allkeys(N): db[k] # ret["lmdbm"]["read"] = t.get() print("lmdbm cont read", N, t.get()) with MeasureTime() as t: with JsonLmdb.open(LMDBM_FILE, "r") as db: for k in randkeys(N, N): db[k] ret["lmdbm"]["read"] = t.get() print("lmdbm rand read", N, t.get()) with open(os.devnull, "w") as devnull: # mute annoying "free lines" output with redirect_stdout(devnull): with MeasureTime() as t: db = pysos.Dict(PYSOS_FILE) for k in randkeys(N, N): db[k] db.close() ret["pysos"]["read"] = t.get() print("pysos read", N, t.get()) with MeasureTime() as t: with SqliteDict(SQLITEDICT_FILE) as db: for k in randkeys(N, N): db[k] ret["sqlitedict"]["read"] = t.get() print("sqlitedict read", N, t.get()) with MeasureTime() as t: with dbm.dumb.open(DBM_DUMB_FILE, "r") as db: for k in randkeys(N, N): json.loads(db[k]) ret["dbm.dumb"]["read"] = t.get() print("dbm.dumb read", N, t.get()) if gdbm: with MeasureTime() as t: with dbm.gnu.open(DBM_GNU_FILE, "r") as db: for k in randkeys(N, N): json.loads(db[k]) ret["dbm.gnu"]["read"] = t.get() print("dbm.gnu read", N, t.get()) with MeasureTime() as t: db = semidbm.open(SEMIDBM_FILE, "r") for k in randkeys(N, N): json.loads(db[k]) db.close() ret["semidbm"]["read"] = t.get() print("semidbm read", N, t.get()) with MeasureTime() as t: with Vedis(VEDIS_FILE) as db: for k in randkeys(N, N): json.loads(db[k]) ret["vedis"]["read"] = t.get() print("vedis read", N, t.get()) with MeasureTime() as t: with UnQLite(UNQLITE_FILE) as db: for k in randkeys(N, N): json.loads(db[k]) ret["unqlite"]["read"] = t.get() print("unqlite read", N, t.get()) return ret
import time import test_rnd as rnd import random import pysos # initialize the data N = 1234 items = [(rnd.utf8(20), rnd.utf8(200)) for i in range(N)] start = time.time() db = pysos.Dict('temp/sos_dict') #import shelve #db = shelve.open('temp.shelve') print("%.2fs: %d items loaded" % (time.time() - start, len(db))) # add all items for key,val in items: db[key] = val print("%.2fs: %d items added" % (time.time() - start, len(items))) # read all keys random.shuffle(items) for key,val in items: val2 = db[key] assert val2 == val print("%.2fs: %d items read" % (time.time() - start, len(items)))