def create_and_populate_client():
client = LocalClient(cache_options, cache_pfx)
# Monkey with the internals so we don't have to
# populate multiple times.
#print("Testing", type(bucket._dict))
# Only need to populate in the workers.
size_dists = [100] * 800 + [300] * 500 + [1024] * 300 + [2048] * 200 + [4096] * 150
with open('/dev/urandom', 'rb') as rnd:
data = [rnd.read(x) for x in size_dists]
data_iter = itertools.cycle(data)
keys_and_values = []
len_values = 0
j = 0
for j, datum in enumerate(data_iter):
if len(datum) > client.limit or len_values + len(datum) > client.limit:
break
len_values += len(datum)
# To ensure the pickle memo cache doesn't just write out "use object X",
# but distinct copies of the strings, we need to copy them
keys_and_values.append(
((j, j), (datum[:-1] + b'x', j))
)
# # We need to get the item so its frequency goes up enough to be written
# # (this is while we're doing an aging at write time, which may go away).
# # Using an assert statement causes us to write nothing if -O is used.
# if bucket[(j, j)] is datum:
# raise AssertionError()
mem_before = get_memory_usage()
client._bulk_update(keys_and_values, mem_usage_before=mem_before)
del keys_and_values
#print("Len", len(bucket), "size", bucket.size, "checkpoints", client.get_checkpoints())
return client
def makeOne(populate=True, data=None):
mem_before = get_memory_usage()
gc.collect()
gc.collect()
objects_before = len(gc.get_objects())
client = LocalClient(options, 'pfx')
client.b_mem_before = mem_before
client.b_objects_before = objects_before
if populate:
client._bulk_update([
(t[0], (t[1][0], t[1][1], False, 1))
for t in data or _make_data(random_data, KEY_GROUP_SIZE)
])
return client
