def create_and_populate_client(): client = LocalClient(cache_options, cache_pfx) # Monkey with the internals so we don't have to # populate multiple times. #print("Testing", type(bucket._dict)) # Only need to populate in the workers. size_dists = [100] * 800 + [300] * 500 + [1024] * 300 + [2048] * 200 + [4096] * 150 with open('/dev/urandom', 'rb') as rnd: data = [rnd.read(x) for x in size_dists] data_iter = itertools.cycle(data) keys_and_values = [] len_values = 0 j = 0 for j, datum in enumerate(data_iter): if len(datum) > client.limit or len_values + len(datum) > client.limit: break len_values += len(datum) # To ensure the pickle memo cache doesn't just write out "use object X", # but distinct copies of the strings, we need to copy them keys_and_values.append( ((j, j), (datum[:-1] + b'x', j)) ) # # We need to get the item so its frequency goes up enough to be written # # (this is while we're doing an aging at write time, which may go away). # # Using an assert statement causes us to write nothing if -O is used. # if bucket[(j, j)] is datum: # raise AssertionError() mem_before = get_memory_usage() client._bulk_update(keys_and_values, mem_usage_before=mem_before) del keys_and_values #print("Len", len(bucket), "size", bucket.size, "checkpoints", client.get_checkpoints()) return client
def makeOne(populate=True, data=None): mem_before = get_memory_usage() gc.collect() gc.collect() objects_before = len(gc.get_objects()) client = LocalClient(options, 'pfx') client.b_mem_before = mem_before client.b_objects_before = objects_before if populate: client._bulk_update([ (t[0], (t[1][0], t[1][1], False, 1)) for t in data or _make_data(random_data, KEY_GROUP_SIZE) ]) return client