def load(): b2 = LocalClientBucket(bucket.limit) _Loader.load_local_cache(cache_options, cache_pfx, b2)
def save_load_benchmark(): from relstorage.cache import LocalClientBucket, _Loader from io import BytesIO import os import itertools import sys sys.setrecursionlimit(500000) bucket = LocalClientBucket(500 * 1024 * 1024) print("Testing", type(bucket._dict)) size_dists = [100] * 800 + [300] * 500 + [1024] * 300 + [2048] * 200 + [ 4096 ] * 150 with open('/dev/urandom', 'rb') as rnd: data = [rnd.read(x) for x in size_dists] data_iter = itertools.cycle(data) for j, datum in enumerate(data_iter): if len(datum ) > bucket.limit or bucket.size + len(datum) > bucket.limit: break # To ensure the pickle memo cache doesn't just write out "use object X", # but distinct copies of the strings, we need to copy them bucket[str(j)] = datum[:-1] + b'x' assert bucket[str(j)] is not datum #print("Len", len(bucket), "size", bucket.size, "dlen", len(datum)) print("Len", len(bucket), "size", bucket.size) number = 1 import timeit import statistics import cProfile import pstats cache_pfx = "pfx" cache_options = MockOptions() cache_options.cache_local_dir = '/tmp' cache_options.cache_local_dir_compress = False def write(): fname = _Loader.save_local_cache(cache_options, cache_pfx, bucket) os.remove(fname) def load(): b2 = LocalClientBucket(bucket.limit) _Loader.load_local_cache(cache_options, cache_pfx, b2) #write_timer = timeit.Timer(write) #write_times = write_timer.repeat(number=number) #print("write average", statistics.mean(write_times), "stddev", statistics.stdev(write_times)) #read_timer = timeit.Timer(load) #read_times = read_timer.repeat(number=number) #print("read average", statistics.mean(read_times), "stddev", statistics.stdev(read_times)) #pr = cProfile.Profile() #pr.enable() fname = _Loader.save_local_cache(cache_options, cache_pfx, bucket) print("Saved to", fname) #pr.disable() #ps = pstats.Stats(pr).sort_stats('cumulative') #ps.print_stats() #return pr = cProfile.Profile() pr.enable() _Loader.load_local_cache(cache_options, cache_pfx, LocalClientBucket(bucket.limit)) pr.disable() ps = pstats.Stats(pr).sort_stats('cumulative') ps.print_stats(.4)