def test_mem_leaks(self):
import gc
from calibre.utils.mem import get_memory as memory
m = Matcher(['a'], scorer=CScorer)
m('a')
def doit(c):
m = Matcher([
c + 'im/one.gif',
c + 'im/two.gif',
c + 'text/one.html',
],
scorer=CScorer)
m('one')
start = memory()
for i in range(10):
doit(unicode_type(i))
gc.collect()
used10 = memory() - start
start = memory()
for i in range(100):
doit(unicode_type(i))
gc.collect()
used100 = memory() - start
if used100 > 0 and used10 > 0:
self.assertLessEqual(used100, 2 * used10)
def test_for_mem_leak():
from calibre.utils.mem import memory, gc_histogram, diff_hists
import gc
gc.disable()
scanner = DeviceScanner()
scanner.scan()
memory() # load the psutil library
for i in xrange(3):
gc.collect()
for reps in (1, 10, 100, 1000):
for i in xrange(3):
gc.collect()
h1 = gc_histogram()
startmem = memory()
for i in xrange(reps):
scanner.scan()
for i in xrange(3):
gc.collect()
usedmem = memory(startmem)
prints('Memory used in %d repetitions of scan(): %.5f KB'%(reps,
1024*usedmem))
prints('Differences in python object counts:')
diff_hists(h1, gc_histogram())
prints()
def test_basic():
stream = BytesIO(b"Rar!\x1a\x07\x00\xcf\x90s\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x14\xe7z\x00\x80#\x00\x17\x00\x00\x00\r\x00\x00\x00\x03\xc2\xb3\x96o\x00\x00\x00\x00\x1d3\x03\x00\x00\x00\x00\x00CMT\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe6h\x04\x17\xff\xcd\x0f\xffk9b\x11]^\x80\xd3dt \x90+\x00\x14\x00\x00\x00\x08\x00\x00\x00\x03\xf1\x84\x93\\\xb9]yA\x1d3\t\x00\xa4\x81\x00\x001\\sub-one\x00\xc0\x0c\x00\x8f\xec\x89\xfe.JM\x86\x82\x0c_\xfd\xfd\xd7\x11\x1a\xef@\x9eHt \x80'\x00\x0e\x00\x00\x00\x04\x00\x00\x00\x03\x9f\xa8\x17\xf8\xaf]yA\x1d3\x07\x00\xa4\x81\x00\x00one.txt\x00\x08\xbf\x08\xae\xf3\xca\x87\xfeo\xfe\xd2n\x80-Ht \x82:\x00\x18\x00\x00\x00\x10\x00\x00\x00\x03\xa86\x81\xdf\xf9fyA\x1d3\x1a\x00\xa4\x81\x00\x00\xe8\xaf\xb6\xe6\xaf\x94\xe5\xb1\x81.txt\x00\x8bh\xf6\xd4kA\\.\x00txt\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe2l\x91\x189\xff\xdf\xfe\xc2\xd3:g\x9a\x19F=cYt \x928\x00\x11\x00\x00\x00\x08\x00\x00\x00\x03\x7f\xd6\xb6\x7f\xeafyA\x1d3\x16\x00\xa4\x81\x00\x00F\xc3\xbc\xc3\x9fe.txt\x00\x01\x00F\xfc\xdfe\x00.txt\x00\xc0<D\xfe\xc8\xef\xbc\xd1\x04I?\xfd\xff\xdbF)]\xe8\xb9\xe1t \x90/\x00\x13\x00\x00\x00\x08\x00\x00\x00\x03\x1a$\x932\xc2]yA\x1d3\r\x00\xa4\x81\x00\x002\\sub-two.txt\x00\xc0\x10\x00S\xec\xcb\x7f\x8b\xa5(\x0b\x01\xcb\xef\xdf\xf6t\x89\x97z\x0eft \x90)\x00\r\x00\x00\x00\r\x00\x00\x00\x03c\x89K\xd3\xc8fyA\x140\x07\x00\xff\xa1\x00\x00symlink\x00\xc02/sub-two.txt\xeb\x86t\xe0\x90#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\xb9]yA\x140\x01\x00\xedA\x00\x001\x00\xc0\xe0Dt\xe0\x90#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\xc2]yA\x140\x01\x00\xedA\x00\x002\x00\xc0u\xa1t \x80,\x00\r\x00\x00\x00\r\x00\x00\x00\x03T\xea\x04\xca\xe6\x84yA\x140\x0c\x00\xa4\x81\x00\x00uncompresseduncompressed\n\xda\x10t \x900\x00\x0e\x00\x00\x00\x04\x00\x00\x00\x035K.\xa6\x18\x85yA\x1d5\x0e\x00\xa4\x81\x00\x00max-compressed\x00\xc0\x00\x08\xbf\x08\xae\xf2\xcc\x01s\xf8\xff\xec\x96\xe8\xc4={\x00@\x07\x00")
tdata = {u'1': b'',
u'1/sub-one': b'sub-one\n',
u'2': b'',
u'2/sub-two.txt': b'sub-two\n',
u'F\xfc\xdfe.txt': b'unicode\n',
u'max-compressed': b'max\n',
u'one.txt': b'one\n',
u'symlink': b'2/sub-two.txt',
u'uncompressed': b'uncompressed\n',
u'\u8bf6\u6bd4\u5c41.txt': b'chinese unicode\n'}
f = RARFile(stream, True)
names = set()
data = {}
if f.comment != b'some comment\n':
raise ValueError('Comment not read: %r != %r'%(
f.comment, b'some comment\n'))
while True:
try:
h = f.current_item
except EOFError:
break
isdir = h['is_directory']
if isdir and h['filename'] not in {'1', '2'}:
raise ValueError('Incorrect identification of a directory')
if h['is_symlink'] and h['filename'] != 'symlink':
raise ValueError('Incorrect identification of a symlink')
names.add(h['filename'])
et = BytesIO()
f.process_current_item(et)
data[h['filename']] = et.getvalue()
if names != {'1/sub-one', 'one.txt', '2/sub-two.txt',
'1', '2', '诶比屁.txt', 'Füße.txt', 'symlink',
'uncompressed', 'max-compressed'}:
raise ValueError('Name list does not match')
if data != tdata:
raise ValueError('Some data was not read correctly')
from calibre.utils.mem import memory
import gc
del f
for i in xrange(3): gc.collect()
num = 300
start = memory()
s = SaveStream(stream)
for i in xrange(num):
with s:
f = RARFile(stream)
f.test()
del f
del s
for i in xrange(3): gc.collect()
used = memory() - start
if used > 1 and not isosx:
raise ValueError('Leaked %s MB for %d calls'%(used, num))
def check_for_mem_leak(self):
import gc
from calibre.utils.mem import memory
memory()
for num in (1, 10, 100):
start = memory()
for i in xrange(num):
self()
for i in xrange(3): gc.collect()
print 'Mem consumption increased by:', memory() - start, 'MB',
print 'after', num, 'repeats'
def get_mem_use(num):
start = memory()
s = SaveStream(stream)
for i in xrange(num):
with s:
f = RARFile(stream)
f.test()
del f, s
for i in xrange(3):
gc.collect()
return memory() - start
def test_mem():
from calibre.utils.mem import memory
import gc
gc.collect()
start_mem = memory()
raw = P('fonts/liberation/LiberationSerif-Regular.ttf', data=True)
calls = 1000
for i in xrange(calls):
subset(raw, (), (('a', 'z'),))
del raw
for i in xrange(3): gc.collect()
print ('Leaked memory per call:', (memory() - start_mem)/calls*1024, 'KB')
def check_for_mem_leak(self):
import gc
from calibre.utils.mem import memory
memory()
for num in (1, 10, 100):
start = memory()
for i in xrange(num):
self()
for i in xrange(3):
gc.collect()
print 'Mem consumption increased by:', memory() - start, 'MB',
print 'after', num, 'repeats'
def measure_memory_usage(self, repetitions, func, *args, **kwargs):
from calibre.utils.mem import memory
gc.disable()
try:
start_mem = memory()
for i in xrange(repetitions):
func(*args, **kwargs)
for i in xrange(3): gc.collect()
end_mem = memory()
finally:
gc.enable()
return end_mem - start_mem
def measure_memory_usage(self, repetitions, func, *args, **kwargs):
from calibre.utils.mem import memory
gc.disable()
try:
start_mem = memory()
for i in xrange(repetitions):
func(*args, **kwargs)
for i in xrange(3): gc.collect()
end_mem = memory()
finally:
gc.enable()
return end_mem - start_mem
def test_mem():
from calibre.utils.mem import memory
import gc
gc.collect()
start_mem = memory()
raw = P('fonts/liberation/LiberationSerif-Regular.ttf', data=True)
calls = 1000
for i in xrange(calls):
subset(raw, (), (('a', 'z'),))
del raw
for i in xrange(3):
gc.collect()
print ('Leaked memory per call:', (memory() - start_mem)/calls*1024, 'KB')
def test_mem_leaks(self):
import gc
from calibre.utils.mem import get_memory as memory
m = Matcher(['a'], scorer=CScorer)
m('a')
def doit(c):
m = Matcher([c+'im/one.gif', c+'im/two.gif', c+'text/one.html',], scorer=CScorer)
m('one')
start = memory()
for i in xrange(10):
doit(str(i))
gc.collect()
used10 = memory() - start
start = memory()
for i in xrange(100):
doit(str(i))
gc.collect()
used100 = memory() - start
if used100 > 0 and used10 >= 0:
self.assertLessEqual(used100, 2 * used10)
def debug_leak(self):
import gc
from calibre.utils.mem import memory
ref = self.dbref
for i in xrange(3): gc.collect()
if ref() is not None:
print 'DB object alive:', ref()
for r in gc.get_referrers(ref())[:10]:
print r
print
print 'before:', self.before_mem
print 'after:', memory()/1024**2
print
self.dbref = self.before_mem = None
def debug_leak(self):
import gc
from calibre.utils.mem import memory
ref = self.dbref
for i in range(3):
gc.collect()
if ref() is not None:
print('DB object alive:', ref())
for r in gc.get_referrers(ref())[:10]:
print(r)
print()
print('before:', self.before_mem)
print('after:', memory())
print()
self.dbref = self.before_mem = None
def debug_leak(self):
import gc
from calibre.utils.mem import memory
ref = self.dbref
for i in xrange(3):
gc.collect()
if ref() is not None:
print 'DB object alive:', ref()
for r in gc.get_referrers(ref())[:10]:
print r
print
print 'before:', self.before_mem
print 'after:', memory() / 1024**2
print
self.dbref = self.before_mem = None
def debug_leak(self):
import gc
from calibre.utils.mem import memory
ref = self.dbref
for i in range(3):
gc.collect()
if ref() is not None:
print('DB object alive:', ref())
for r in gc.get_referrers(ref())[:10]:
print(r)
print()
print('before:', self.before_mem)
print('after:', memory())
print()
self.dbref = self.before_mem = None
def test_for_mem_leak():
from calibre.utils.mem import memory, gc_histogram, diff_hists
import gc
gc.disable()
scanner = DeviceScanner()
scanner.scan()
memory() # load the psutil library
for i in xrange(3):
gc.collect()
for reps in (1, 10, 100, 1000):
for i in xrange(3):
gc.collect()
h1 = gc_histogram()
startmem = memory()
for i in xrange(reps):
scanner.scan()
for i in xrange(3):
gc.collect()
usedmem = memory(startmem)
prints('Memory used in %d repetitions of scan(): %.5f KB' %
(reps, 1024 * usedmem))
prints('Differences in python object counts:')
diff_hists(h1, gc_histogram())
prints()
if not iswindows:
return
for reps in (1, 10, 100, 1000):
for i in xrange(3):
gc.collect()
h1 = gc_histogram()
startmem = memory()
for i in xrange(reps):
win_pnp_drives()
for i in xrange(3):
gc.collect()
usedmem = memory(startmem)
prints('Memory used in %d repetitions of pnp_scan(): %.5f KB' %
(reps, 1024 * usedmem))
prints('Differences in python object counts:')
diff_hists(h1, gc_histogram())
prints()
