def test_mem_leaks(self): import gc from calibre.utils.mem import get_memory as memory m = Matcher(['a'], scorer=CScorer) m('a') def doit(c): m = Matcher([ c + 'im/one.gif', c + 'im/two.gif', c + 'text/one.html', ], scorer=CScorer) m('one') start = memory() for i in range(10): doit(unicode_type(i)) gc.collect() used10 = memory() - start start = memory() for i in range(100): doit(unicode_type(i)) gc.collect() used100 = memory() - start if used100 > 0 and used10 > 0: self.assertLessEqual(used100, 2 * used10)
def test_for_mem_leak(): from calibre.utils.mem import memory, gc_histogram, diff_hists import gc gc.disable() scanner = DeviceScanner() scanner.scan() memory() # load the psutil library for i in xrange(3): gc.collect() for reps in (1, 10, 100, 1000): for i in xrange(3): gc.collect() h1 = gc_histogram() startmem = memory() for i in xrange(reps): scanner.scan() for i in xrange(3): gc.collect() usedmem = memory(startmem) prints('Memory used in %d repetitions of scan(): %.5f KB'%(reps, 1024*usedmem)) prints('Differences in python object counts:') diff_hists(h1, gc_histogram()) prints()
def test_basic(): stream = BytesIO(b"Rar!\x1a\x07\x00\xcf\x90s\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x14\xe7z\x00\x80#\x00\x17\x00\x00\x00\r\x00\x00\x00\x03\xc2\xb3\x96o\x00\x00\x00\x00\x1d3\x03\x00\x00\x00\x00\x00CMT\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe6h\x04\x17\xff\xcd\x0f\xffk9b\x11]^\x80\xd3dt \x90+\x00\x14\x00\x00\x00\x08\x00\x00\x00\x03\xf1\x84\x93\\\xb9]yA\x1d3\t\x00\xa4\x81\x00\x001\\sub-one\x00\xc0\x0c\x00\x8f\xec\x89\xfe.JM\x86\x82\x0c_\xfd\xfd\xd7\x11\x1a\xef@\x9eHt \x80'\x00\x0e\x00\x00\x00\x04\x00\x00\x00\x03\x9f\xa8\x17\xf8\xaf]yA\x1d3\x07\x00\xa4\x81\x00\x00one.txt\x00\x08\xbf\x08\xae\xf3\xca\x87\xfeo\xfe\xd2n\x80-Ht \x82:\x00\x18\x00\x00\x00\x10\x00\x00\x00\x03\xa86\x81\xdf\xf9fyA\x1d3\x1a\x00\xa4\x81\x00\x00\xe8\xaf\xb6\xe6\xaf\x94\xe5\xb1\x81.txt\x00\x8bh\xf6\xd4kA\\.\x00txt\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe2l\x91\x189\xff\xdf\xfe\xc2\xd3:g\x9a\x19F=cYt \x928\x00\x11\x00\x00\x00\x08\x00\x00\x00\x03\x7f\xd6\xb6\x7f\xeafyA\x1d3\x16\x00\xa4\x81\x00\x00F\xc3\xbc\xc3\x9fe.txt\x00\x01\x00F\xfc\xdfe\x00.txt\x00\xc0<D\xfe\xc8\xef\xbc\xd1\x04I?\xfd\xff\xdbF)]\xe8\xb9\xe1t \x90/\x00\x13\x00\x00\x00\x08\x00\x00\x00\x03\x1a$\x932\xc2]yA\x1d3\r\x00\xa4\x81\x00\x002\\sub-two.txt\x00\xc0\x10\x00S\xec\xcb\x7f\x8b\xa5(\x0b\x01\xcb\xef\xdf\xf6t\x89\x97z\x0eft \x90)\x00\r\x00\x00\x00\r\x00\x00\x00\x03c\x89K\xd3\xc8fyA\x140\x07\x00\xff\xa1\x00\x00symlink\x00\xc02/sub-two.txt\xeb\x86t\xe0\x90#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\xb9]yA\x140\x01\x00\xedA\x00\x001\x00\xc0\xe0Dt\xe0\x90#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\xc2]yA\x140\x01\x00\xedA\x00\x002\x00\xc0u\xa1t \x80,\x00\r\x00\x00\x00\r\x00\x00\x00\x03T\xea\x04\xca\xe6\x84yA\x140\x0c\x00\xa4\x81\x00\x00uncompresseduncompressed\n\xda\x10t \x900\x00\x0e\x00\x00\x00\x04\x00\x00\x00\x035K.\xa6\x18\x85yA\x1d5\x0e\x00\xa4\x81\x00\x00max-compressed\x00\xc0\x00\x08\xbf\x08\xae\xf2\xcc\x01s\xf8\xff\xec\x96\xe8\xc4={\x00@\x07\x00") tdata = {u'1': b'', u'1/sub-one': b'sub-one\n', u'2': b'', u'2/sub-two.txt': b'sub-two\n', u'F\xfc\xdfe.txt': b'unicode\n', u'max-compressed': b'max\n', u'one.txt': b'one\n', u'symlink': b'2/sub-two.txt', u'uncompressed': b'uncompressed\n', u'\u8bf6\u6bd4\u5c41.txt': b'chinese unicode\n'} f = RARFile(stream, True) names = set() data = {} if f.comment != b'some comment\n': raise ValueError('Comment not read: %r != %r'%( f.comment, b'some comment\n')) while True: try: h = f.current_item except EOFError: break isdir = h['is_directory'] if isdir and h['filename'] not in {'1', '2'}: raise ValueError('Incorrect identification of a directory') if h['is_symlink'] and h['filename'] != 'symlink': raise ValueError('Incorrect identification of a symlink') names.add(h['filename']) et = BytesIO() f.process_current_item(et) data[h['filename']] = et.getvalue() if names != {'1/sub-one', 'one.txt', '2/sub-two.txt', '1', '2', '诶比屁.txt', 'Füße.txt', 'symlink', 'uncompressed', 'max-compressed'}: raise ValueError('Name list does not match') if data != tdata: raise ValueError('Some data was not read correctly') from calibre.utils.mem import memory import gc del f for i in xrange(3): gc.collect() num = 300 start = memory() s = SaveStream(stream) for i in xrange(num): with s: f = RARFile(stream) f.test() del f del s for i in xrange(3): gc.collect() used = memory() - start if used > 1 and not isosx: raise ValueError('Leaked %s MB for %d calls'%(used, num))
def check_for_mem_leak(self): import gc from calibre.utils.mem import memory memory() for num in (1, 10, 100): start = memory() for i in xrange(num): self() for i in xrange(3): gc.collect() print 'Mem consumption increased by:', memory() - start, 'MB', print 'after', num, 'repeats'
def get_mem_use(num): start = memory() s = SaveStream(stream) for i in xrange(num): with s: f = RARFile(stream) f.test() del f, s for i in xrange(3): gc.collect() return memory() - start
def test_mem(): from calibre.utils.mem import memory import gc gc.collect() start_mem = memory() raw = P('fonts/liberation/LiberationSerif-Regular.ttf', data=True) calls = 1000 for i in xrange(calls): subset(raw, (), (('a', 'z'),)) del raw for i in xrange(3): gc.collect() print ('Leaked memory per call:', (memory() - start_mem)/calls*1024, 'KB')
def measure_memory_usage(self, repetitions, func, *args, **kwargs): from calibre.utils.mem import memory gc.disable() try: start_mem = memory() for i in xrange(repetitions): func(*args, **kwargs) for i in xrange(3): gc.collect() end_mem = memory() finally: gc.enable() return end_mem - start_mem
def test_mem_leaks(self): import gc from calibre.utils.mem import get_memory as memory m = Matcher(['a'], scorer=CScorer) m('a') def doit(c): m = Matcher([c+'im/one.gif', c+'im/two.gif', c+'text/one.html',], scorer=CScorer) m('one') start = memory() for i in xrange(10): doit(str(i)) gc.collect() used10 = memory() - start start = memory() for i in xrange(100): doit(str(i)) gc.collect() used100 = memory() - start if used100 > 0 and used10 >= 0: self.assertLessEqual(used100, 2 * used10)
def debug_leak(self): import gc from calibre.utils.mem import memory ref = self.dbref for i in xrange(3): gc.collect() if ref() is not None: print 'DB object alive:', ref() for r in gc.get_referrers(ref())[:10]: print r print print 'before:', self.before_mem print 'after:', memory()/1024**2 print self.dbref = self.before_mem = None
def debug_leak(self): import gc from calibre.utils.mem import memory ref = self.dbref for i in range(3): gc.collect() if ref() is not None: print('DB object alive:', ref()) for r in gc.get_referrers(ref())[:10]: print(r) print() print('before:', self.before_mem) print('after:', memory()) print() self.dbref = self.before_mem = None
def debug_leak(self): import gc from calibre.utils.mem import memory ref = self.dbref for i in xrange(3): gc.collect() if ref() is not None: print 'DB object alive:', ref() for r in gc.get_referrers(ref())[:10]: print r print print 'before:', self.before_mem print 'after:', memory() / 1024**2 print self.dbref = self.before_mem = None
def test_for_mem_leak(): from calibre.utils.mem import memory, gc_histogram, diff_hists import gc gc.disable() scanner = DeviceScanner() scanner.scan() memory() # load the psutil library for i in xrange(3): gc.collect() for reps in (1, 10, 100, 1000): for i in xrange(3): gc.collect() h1 = gc_histogram() startmem = memory() for i in xrange(reps): scanner.scan() for i in xrange(3): gc.collect() usedmem = memory(startmem) prints('Memory used in %d repetitions of scan(): %.5f KB' % (reps, 1024 * usedmem)) prints('Differences in python object counts:') diff_hists(h1, gc_histogram()) prints() if not iswindows: return for reps in (1, 10, 100, 1000): for i in xrange(3): gc.collect() h1 = gc_histogram() startmem = memory() for i in xrange(reps): win_pnp_drives() for i in xrange(3): gc.collect() usedmem = memory(startmem) prints('Memory used in %d repetitions of pnp_scan(): %.5f KB' % (reps, 1024 * usedmem)) prints('Differences in python object counts:') diff_hists(h1, gc_histogram()) prints()