def test_tempfile(self):
'''repeatedly seralize to temp file, then repeatedly deserialize from
it, checking usage all along the way.
'''
if cdump is None:
logger.warn('no C dump(), skipping test_tempfile')
return
with tempfile.NamedTemporaryFile() as ntf:
# first, write a bunch to temp file
with open(ntf.name, 'wb') as fout:
sys.stderr.write('write {!r} {}\n'.format(ntf.name, fout))
start_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history = [start_usage]
for o in _range(_TEST_OUTER):
for i in _range(_TEST_COUNT):
ob = _randob()
cdump(ob, fout)
t_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history.append(t_usage)
end_usage = usage_history[-1]
dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
didrss = end_usage.ru_idrss - start_usage.ru_idrss
dmaxrsspct = ((end_usage.ru_maxrss != 0) and (dmaxrss / end_usage.ru_maxrss)) or 0
didrsspct = ((end_usage.ru_idrss != 0) and (didrss / end_usage.ru_idrss)) or 0
sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss, dmaxrsspct * 100.0))
sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_idrss, end_usage.ru_idrss, didrss, didrsspct * 100.0))
assert (dmaxrsspct) < 0.05, [x.ru_maxrss for x in usage_history]
assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]
sys.stderr.write('{!r} is {} bytes\n'.format(ntf.name, os.path.getsize(ntf.name)))
# now, read a bunch back from temp file.
with open(ntf.name, 'rb') as fin:
sys.stderr.write('read {!r} {}\n'.format(ntf.name, fin))
start_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history = [start_usage]
for o in _range(_TEST_OUTER):
for i in _range(_TEST_COUNT):
dob = cload(fin)
# and silently drop the result. I hope the garbage collector works!
gc.collect()
t_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history.append(t_usage)
end_usage = usage_history[-1]
dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
didrss = end_usage.ru_idrss - start_usage.ru_idrss
dmaxrsspct = ((end_usage.ru_maxrss != 0) and (dmaxrss / end_usage.ru_maxrss)) or 0
didrsspct = ((end_usage.ru_idrss != 0) and (didrss / end_usage.ru_idrss)) or 0
sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss, dmaxrsspct * 100.0))
sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_idrss, end_usage.ru_idrss, didrss, didrsspct * 100.0))
assert (dmaxrsspct) < 0.05, [x.ru_maxrss for x in usage_history]
assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]
def test_stringio_usage(self):
'''serialize data to StringIO, read it back'''
if cdump is None:
logger.warn('no C dump(), skipping test_tempfile')
return
# warmup the rusage, allocate everything!
fout = StringIO()
sys.stderr.write('write 1 to StringIO\n')
oblist = []
for o in _range(_TEST_OUTER):
for i in _range(_TEST_COUNT):
ob = _randob()
oblist.append(ob)
cdump(ob, fout)
# position at start to overwrite, but leave allocated
fout.seek(0)
sys.stderr.write('write 2 to StringIO\n')
start_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history = [start_usage]
pos = 0
for o in _range(_TEST_OUTER):
for i in _range(_TEST_COUNT):
ob = oblist[pos]
pos += 1
cdump(ob, fout)
gc.collect()
t_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history.append(t_usage)
end_usage = usage_history[-1]
dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
didrss = end_usage.ru_idrss - start_usage.ru_idrss
dmaxrsspct = ((end_usage.ru_maxrss != 0) and
(dmaxrss / end_usage.ru_maxrss)) or 0
didrsspct = ((end_usage.ru_idrss != 0) and
(didrss / end_usage.ru_idrss)) or 0
sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(
start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss,
dmaxrsspct * 100.0))
sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(
start_usage.ru_idrss, end_usage.ru_idrss, didrss,
didrsspct * 100.0))
assert (dmaxrsspct) < 0.05, [x.ru_maxrss for x in usage_history]
assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]
sys.stderr.write('StringIO is {} bytes\n'.format(fout.tell()))
fout.seek(0)
fin = fout
sys.stderr.write('read StringIO\n')
start_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history = [start_usage]
for o in _range(_TEST_OUTER):
for i in _range(_TEST_COUNT):
dob = cload(fin)
# and silently drop the result. I hope the garbage collector works!
gc.collect()
t_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history.append(t_usage)
end_usage = usage_history[-1]
dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
didrss = end_usage.ru_idrss - start_usage.ru_idrss
dmaxrsspct = ((end_usage.ru_maxrss != 0) and
(dmaxrss / end_usage.ru_maxrss)) or 0
didrsspct = ((end_usage.ru_idrss != 0) and
(didrss / end_usage.ru_idrss)) or 0
sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(
start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss,
dmaxrsspct * 100.0))
sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(
start_usage.ru_idrss, end_usage.ru_idrss, didrss,
didrsspct * 100.0))
assert (dmaxrsspct) < 0.05, [x.ru_maxrss for x in usage_history]
assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]
def test_tempfile(self):
'''repeatedly seralize to temp file, then repeatedly deserialize from
it, checking usage all along the way.
'''
if cdump is None:
logger.warn('no C dump(), skipping test_tempfile')
return
with tempfile.NamedTemporaryFile() as ntf:
# first, write a bunch to temp file
with open(ntf.name, 'wb') as fout:
sys.stderr.write('write {!r} {}\n'.format(ntf.name, fout))
start_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history = [start_usage]
for o in _range(_TEST_OUTER):
for i in _range(_TEST_COUNT):
ob = _randob()
cdump(ob, fout)
t_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history.append(t_usage)
end_usage = usage_history[-1]
dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
didrss = end_usage.ru_idrss - start_usage.ru_idrss
dmaxrsspct = ((end_usage.ru_maxrss != 0) and
(dmaxrss / end_usage.ru_maxrss)) or 0
didrsspct = ((end_usage.ru_idrss != 0) and
(didrss / end_usage.ru_idrss)) or 0
sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(
start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss,
dmaxrsspct * 100.0))
sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(
start_usage.ru_idrss, end_usage.ru_idrss, didrss,
didrsspct * 100.0))
assert (dmaxrsspct) < 0.05, [
x.ru_maxrss for x in usage_history
]
assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]
sys.stderr.write('{!r} is {} bytes\n'.format(
ntf.name, os.path.getsize(ntf.name)))
# now, read a bunch back from temp file.
with open(ntf.name, 'rb') as fin:
sys.stderr.write('read {!r} {}\n'.format(ntf.name, fin))
start_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history = [start_usage]
for o in _range(_TEST_OUTER):
for i in _range(_TEST_COUNT):
dob = cload(fin)
# and silently drop the result. I hope the garbage collector works!
gc.collect()
t_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history.append(t_usage)
end_usage = usage_history[-1]
dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
didrss = end_usage.ru_idrss - start_usage.ru_idrss
dmaxrsspct = ((end_usage.ru_maxrss != 0) and
(dmaxrss / end_usage.ru_maxrss)) or 0
didrsspct = ((end_usage.ru_idrss != 0) and
(didrss / end_usage.ru_idrss)) or 0
sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(
start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss,
dmaxrsspct * 100.0))
sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(
start_usage.ru_idrss, end_usage.ru_idrss, didrss,
didrsspct * 100.0))
assert (dmaxrsspct) < 0.05, [
x.ru_maxrss for x in usage_history
]
assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]
def test_stringio_usage(self):
'''serialize data to StringIO, read it back'''
if cdump is None:
logger.warn('no C dump(), skipping test_tempfile')
return
# warmup the rusage, allocate everything!
fout = StringIO()
sys.stderr.write('write 1 to StringIO\n')
oblist = []
for o in _range(_TEST_OUTER):
for i in _range(_TEST_COUNT):
ob = _randob()
oblist.append(ob)
cdump(ob, fout)
# position at start to overwrite, but leave allocated
fout.seek(0)
sys.stderr.write('write 2 to StringIO\n')
start_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history = [start_usage]
pos = 0
for o in _range(_TEST_OUTER):
for i in _range(_TEST_COUNT):
ob = oblist[pos]
pos += 1
cdump(ob, fout)
gc.collect()
t_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history.append(t_usage)
end_usage = usage_history[-1]
dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
didrss = end_usage.ru_idrss - start_usage.ru_idrss
dmaxrsspct = ((end_usage.ru_maxrss != 0) and (dmaxrss / end_usage.ru_maxrss)) or 0
didrsspct = ((end_usage.ru_idrss != 0) and (didrss / end_usage.ru_idrss)) or 0
sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss, dmaxrsspct * 100.0))
sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_idrss, end_usage.ru_idrss, didrss, didrsspct * 100.0))
assert (dmaxrsspct) < 0.05, [x.ru_maxrss for x in usage_history]
assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]
sys.stderr.write('StringIO is {} bytes\n'.format(fout.tell()))
fout.seek(0)
fin = fout
sys.stderr.write('read StringIO\n')
start_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history = [start_usage]
for o in _range(_TEST_OUTER):
for i in _range(_TEST_COUNT):
dob = cload(fin)
# and silently drop the result. I hope the garbage collector works!
gc.collect()
t_usage = resource.getrusage(resource.RUSAGE_SELF)
usage_history.append(t_usage)
end_usage = usage_history[-1]
dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
didrss = end_usage.ru_idrss - start_usage.ru_idrss
dmaxrsspct = ((end_usage.ru_maxrss != 0) and (dmaxrss / end_usage.ru_maxrss)) or 0
didrsspct = ((end_usage.ru_idrss != 0) and (didrss / end_usage.ru_idrss)) or 0
sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss, dmaxrsspct * 100.0))
sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_idrss, end_usage.ru_idrss, didrss, didrsspct * 100.0))
assert (dmaxrsspct) < 0.05, [x.ru_maxrss for x in usage_history]
assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]