def log_mem_usage(signum, frame, fname=None):
global _count
_count += 1
gc.collect()
if not fname:
fname = filename + '_memory_%02d.log' % _count
with open(fname, 'wb') as f:
f.write('gc.garbage: %d\n\n' % len(gc.garbage))
objgraph.show_most_common_types(limit=50, file=f)
f.write('\n\n')
buf = StringIO()
objgraph.show_growth(limit=50, file=buf)
buf = buf.getvalue()
f.write(buf)
if _count < 2:
return
for tn, l in enumerate(buf.splitlines()[:10]):
l = l.strip()
if not l:
continue
type_ = l.split()[0]
objects = objgraph.by_type(type_)
objects = random.sample(objects, min(50, len(objects)))
objgraph.show_chain(
objgraph.find_backref_chain(
objects[0],
objgraph.is_proper_module),
filename=fname[:-4] + '_type_%02d_backref.png' % tn
)
objgraph.show_backrefs(
objects,
max_depth=5,
extra_info=lambda x: hex(id(x)),
filename=fname[:-4] + '_type_%02d_backrefs.png' % tn,
)
def trace_memory_stop(self):
""" Stops measuring memory consumption """
objgraph.show_growth(limit=10)
trace_type = get_current_config()["trace_memory_type"]
if trace_type:
objgraph.show_chain(
objgraph.find_backref_chain(
random.choice(objgraph.by_type(trace_type)),
objgraph.is_proper_module
),
filename='%s/%s-%s.png' % (get_current_config()["trace_memory_output_dir"], trace_type, self.id)
)
gc.collect()
self._memory_stop = self.worker.get_memory()
diff = self._memory_stop - self._memory_start
log.debug("Memory diff for job %s : %s" % (self.id, diff))
# We need to update it later than the results, we need them off memory already.
self.collection.update({
"_id": self.id
}, {"$set": {
"memory_diff": diff
}}, w=1)
def get_counts(prefix=""):
test_types = [
"Var",
"Piecewise",
"ScenarioTree",
"ScenarioTreeNode",
"Scenario",
"_SetContainer",
"_ConstraintArray",
]
# use objgraph to check if these pyomo objects still exist
objects = {}
test_counts = {}
for name in test_types:
objects[name] = objgraph.by_type(name)
test_counts[name] = len(objects[name])
if True:
for name in test_types:
if test_counts[name] == 0:
continue
else:
obj = objects[name][0]
fname = prefix + "-" + "objgraph-{}-".format(name)
objgraph.show_refs([obj],
filename=fname + "refs.png") # too_many=50,
objgraph.show_backrefs([obj], filename=fname + "backref.png")
objgraph.show_chain(
objgraph.find_backref_chain(obj, inspect.ismodule),
filename=fname + "chain.png",
)
return test_counts
def show_backref_from_coro():
import random
objgraph.show_chain(
objgraph.find_backref_chain(
random.choice(objgraph.by_type('coroutine')),
objgraph.is_proper_module),
filename='chain-to-leaked.png')
def get_counts(prefix=''):
test_types = [
'Var',
'Piecewise',
'ScenarioTree',
'ScenarioTreeNode',
'Scenario',
'_SetContainer',
'_ConstraintArray',
]
# use objgraph to check if these pyomo objects still exist
objects = {}
test_counts = {}
for name in test_types:
objects[name] = objgraph.by_type( name )
test_counts[name] = len(objects[name])
if True:
for name in test_types:
if test_counts[name]==0: continue
else: obj = objects[name][0]
fname = prefix+'-'+'objgraph-{}-'.format(name)
objgraph.show_refs( [obj], filename=fname+'refs.png') # too_many=50,
objgraph.show_backrefs([obj], filename=fname+'backref.png')
objgraph.show_chain(
objgraph.find_backref_chain( obj, inspect.ismodule),
filename=fname+'chain.png'
)
return test_counts
def log_mem_usage(signum, frame, fname=None):
global _count
_count += 1
gc.collect()
if not fname:
fname = filename + '_memory_%02d.log' % _count
with open(fname, 'wb') as f:
f.write('gc.garbage: %d\n\n' % len(gc.garbage))
objgraph.show_most_common_types(limit=50, file=f)
f.write('\n\n')
buf = StringIO()
objgraph.show_growth(limit=50, file=buf)
buf = buf.getvalue()
f.write(buf)
if _count < 2:
return
for tn, l in enumerate(buf.splitlines()[:10]):
l = l.strip()
if not l:
continue
type_ = l.split()[0]
objects = objgraph.by_type(type_)
objects = random.sample(objects, min(50, len(objects)))
objgraph.show_chain(
objgraph.find_backref_chain(objects[0], objgraph.is_proper_module),
filename=fname[:-4] + '_type_%02d_backref.png' % tn)
objgraph.show_backrefs(
objects,
max_depth=5,
extra_info=lambda x: hex(id(x)),
filename=fname[:-4] + '_type_%02d_backrefs.png' % tn,
)
def print_objects(f):
gc.collect()
with open(f, 'r') as fd:
for line in fd:
line = line.strip()
try:
obj = random.choice(objgraph.by_type(line))
except Exception as e:
LOGGER.info('exception trying to objgraph a random %s: %s',
line, str(e))
break
with tempfile.NamedTemporaryFile(dir='/tmp',
prefix=line,
suffix='.dot',
mode='w') as out:
try:
objgraph.show_chain(objgraph.find_backref_chain(
obj, objgraph.is_proper_module),
output=out)
LOGGER.info('object %s file %s', line, out.name)
except Exception as e:
LOGGER.info(
'exception trying to show_chain a random %s: %s', line,
str(e))
try:
os.remove(f)
except Exception as e:
LOGGER.info('exception %s removing memory_crawler file %s', str(e), f)
def trace_memory_stop(self):
""" Stops measuring memory consumption """
self.trace_memory_clean_caches()
objgraph.show_growth(limit=30)
trace_type = context.get_current_config()["trace_memory_type"]
if trace_type:
filename = '%s/%s-%s.png' % (context.get_current_config(
)["trace_memory_output_dir"], trace_type, self.id)
chain = objgraph.find_backref_chain(
random.choice(objgraph.by_type(trace_type)),
objgraph.is_proper_module)
objgraph.show_chain(chain, filename=filename)
del filename
del chain
gc.collect()
self._memory_stop = self.worker.get_memory()["total"]
diff = self._memory_stop - self._memory_start
context.log.debug("Memory diff for job %s : %s" % (self.id, diff))
# We need to update it later than the results, we need them off memory
# already.
self.collection.update({"_id": self.id},
{"$set": {
"memory_diff": diff
}},
w=1)
def inspect(self):
obj = self._get_request_object()
if obj is None:
return redirect(url_for('.overview'))
# Figure out attributes
attrs = tools.get_public_attrs(obj)
# Backrefs
referrers = objgraph.find_backref_chain(obj, objgraph.is_proper_module,
2)
sorted_referrers = sorted([(self.get_repr(o), o) for o in referrers
if id(o) != id(obj)])
# Refs
referrents = objgraph.find_ref_chain(obj, objgraph.is_proper_module, 2)
sorted_referrents = sorted([(self.get_repr(o), o) for o in referrents
if id(o) != id(obj)])
return self.render('flask-admin-profiler/memory/object.html',
obj=obj,
obj_text=tools.pretty_print(obj),
obj_id=format_id(obj),
obj_type=tools.get_type(obj),
format_id=format_id,
attrs=attrs,
referrers=sorted_referrers,
referrents=sorted_referrents)
def test_single_dm_instance(driver_manager, db):
"""
Our driver manager should only be linked to one PostgresDb instance in memory.
Subsequent drivers should share the same instance to avoid extra connections (and cache duplication)
"""
# There's a circular reference in DriverManager structure, so old test instances aren't cleaned up by
# the reference counter.
gc.collect()
# For all PostgresDb instances in memory (there may be others due to other pytests), count
# how many are connected to our instance of DriverManager
references = 0
for pg_instance in objgraph.by_type('PostgresDb'):
chain = objgraph.find_backref_chain(
pg_instance,
max_depth=10,
predicate=lambda o: isinstance(o, DriverManager))
# If the referenced DriverManager is ours
if chain[0] is driver_manager:
print("Length {}: {}".format(
len(chain), ' -> '.join(c.__class__.__name__ for c in chain)))
references += 1
assert references == 1, "Our DriverManager should only reference one PG instance"
def run_last():
run_once("last")
objgraph.show_chain(objgraph.find_backref_chain(
random.choice(objgraph.by_type('BoundField')),
objgraph.is_proper_module),
filename=r'c:\temp\chain.png')
print("DONE LAST")
def show_chain_of_leaked():
# LeakedOne must be leaked for this to work
import random
func_that_will_leak()
objgraph.show_chain(objgraph.find_backref_chain(
random.choice(objgraph.by_type('LeakedOne')),
objgraph.is_proper_module),
filename='chain-to-leaked.png')
def show_memory_refs(name):
try: obj=objgraph.by_type(name)[0]
except IndexError:
print 'no object of type',name
return
objgraph.show_chain(
objgraph.find_backref_chain(obj, inspect.ismodule),
filename='chain-{}.png'.format(name))
def run_last():
run_once("last")
objgraph.show_chain(
objgraph.find_backref_chain(
random.choice(objgraph.by_type('BoundField')),
objgraph.is_proper_module),
filename=r'c:\temp\chain.png')
print( "DONE LAST")
def new_item_refmaps(collect=True,
types=None,
skip=2,
max_per_type=10,
max_total=50,
tags=None):
log.debug('new_item_refmaps(%s)', collect)
if collect:
log.debug(' forcing garbage collection')
gc.collect()
pid = os.getpid()
log.debug(' finding new_ids')
new_ids = objgraph.get_new_ids() # dict of sets
if STATE['new_item_refmaps'] < skip:
log.debug(' query #%d < %d; not analyzing yet',
STATE['new_item_refmaps'], skip)
STATE['new_item_refmaps'] += 1
else:
done = False
events = list()
if types is None:
types = new_ids.keys()
total_count = 0
for type in types:
type_count = 0
log.debug(' finding backref chains for items of type=%s', type)
for item_id in new_ids.get(type, []):
item = objgraph.at(item_id)
name = _obj_name(item)
log.debug(' finding backrefs chain for item=%s', name)
refmap = objgraph.find_backref_chain(item,
objgraph.is_proper_module)
refmap = [_obj_name(x) for x in refmap]
events.append({
'time': _now(),
'pid': pid,
'type': type,
'name': name,
'chain': refmap
})
total_count += 1
type_count += 1
if type_count > max_per_type:
log.debug('reached max_per_type=%d limit', max_per_type)
done = True
if total_count > max_total:
log.debug('reached max_total=%d limit', max_total)
done = True
if done:
break
if done:
break
if events:
_apply_tags(tags, *events)
return {'sourcetype': 'memleak.new_item_refmaps', 'events': events}
def show_chain(self, obj_type):
# obj_type: Myobj_Type, type:string
ref_chain = objgraph.find_backref_chain(
random.choice(objgraph.by_type(obj_type)),
objgraph.is_proper_module,
max_depth=5)
objgraph.show_chain(ref_chain, filename='chain.dot')
cmd = "dot -Tpng chain.dot -o chain.png"
commands.getstatusoutput(cmd)
def process_response(self, request, response):
objgraph.show_growth(limit=100)
track_class_name = getattr(settings, 'ASPARAGUS_TRACK_CLASS', None)
if track_class_name:
objgraph.show_chain(objgraph.find_backref_chain(
objgraph.by_type(track_class_name)[-1],
objgraph.is_proper_module),
filename='leaks.png')
return response
def backref_graph(self):
obj = self._get_request_object()
if obj is None:
return redirect(url_for('.overview'))
depth = request.args.get('depth', type=int, default=self.MAX_DEPTH)
if depth > self.MAX_DEPTH:
depth = self.MAX_DEPTH
objs = objgraph.find_backref_chain(obj, objgraph.is_proper_module,
depth)
return self._render_ref_graph(objs)
def graph_randobj(line):
import objgraph
args = shlex.split(line)
if len(args) > 1:
fname = args[1]
else:
fname = 'chain.png'
print 'Getting random %s object...' % args[0]
obj = random.choice(objgraph.by_type(args[0]))
print 'Creating chain...'
chain = objgraph.find_backref_chain(obj, objgraph.is_proper_module)
print 'Saving chain...'
objgraph.show_chain(chain, filename=fname)
def graph_randobj(line):
import objgraph
args = shlex.split(line)
if len(args) > 1:
fname = args[1]
else:
fname = 'chain.png'
print 'Getting random %s object...' % args[0]
obj = random.choice(objgraph.by_type(args[0]))
print 'Creating chain...'
chain = objgraph.find_backref_chain(obj, objgraph.is_proper_module)
print 'Saving chain...'
objgraph.show_chain(chain, filename=fname)
def gc_debug_backtraces(magictype=None, count=1):
sio = StringIO.StringIO()
orig_out = sys.stdout
try:
sys.stdout = sio
all_objects = list(objgraph.by_type(magictype))
random.shuffle(all_objects)
chains = []
for obj in all_objects[:count]:
chains.append(
objgraph.find_backref_chain(obj, objgraph.is_proper_module))
sio2 = StringIO.StringIO()
objgraph.show_chain(*chains, output=sio2)
logging.info("%s", sio2.getvalue())
finally:
sys.stdout = orig_out
def dump_memory():
now = time() - start
if PROFILE_MEMORY_GRAPH_BACKREF_TYPES:
for type_ in types:
for sample_number in xrange(
PROFILE_MEMORY_GRAPH_BACKREF_AMOUNT):
objects = objgraph.by_type(type_)
if objects:
objgraph.show_chain(objgraph.find_backref_chain(
random.choice(objects), objgraph.is_proper_module),
filename=objgraph_out_file %
(type_, now, sample_number))
else:
logger.error("No objects of type %s found!", type_)
scanner.dump_all_objects(meliae_out_file % now)
def graphMemProfile(func, args, kw_args):
import objgraph
# objgraph.show_most_common_types()
objgraph.show_growth()
func(*args, **kw_args)
objgraph.show_growth()
import random
str(
objgraph.show_chain(objgraph.find_backref_chain(
random.choice(objgraph.by_type('staticmethod')),
objgraph.is_proper_module),
filename='chain.png'))
#objgraph.show_refs([func(*args,**kw_args)], filename='refs.png')
#objgraph.show_backrefs([func(*args,**kw_args)], filename='backrefs.png')
#import pdb; pdb.set_trace()
objgraph.show_growth()
def backref_objects_by_type(obj_type):
"""Find all of the objects by a given type and graph back-reference sample.
Randomly graph the back-references for a small number of instances.
"""
if LOG.isEnabledFor(logging.DEBUG3) and _ObjGraph:
#
# Any nested function will hold onto some local state and leak.
# Since gc.get_objects() contains the very frame running this code,
# you get a circular reference.
#
gc.collect()
all_objects = gc.get_objects()
try:
objects = objgraph.by_type(obj_type, all_objects)
finally:
del all_objects
LOG.debug3("{} objects of type {} on heap".format(
len(objects), obj_type))
if objects:
try:
obj = random.choice(objects)
finally:
del objects
# prune the unhashables from the garbage list to avoid errors with set()
garbage = [o for o in gc.garbage if is_hashable(o)]
try:
### this seems to interfere with the measurements...
chain = objgraph.find_backref_chain(obj,
inspect.ismodule,
extra_ignore=garbage)
finally:
del garbage
# Delete after calling objgraph, which invokes gc.collect()
del gc.garbage[:]
try:
LOG.debug3("{} chain for {}".format(obj_type, obj))
for link in chain:
LOG.debug3("chain link: {}: {}".format(
type(link),
str(link)[:240]))
finally:
del chain
def print_objects(f):
gc.collect()
with open(f, 'r') as fd:
for line in fd:
line = line.strip()
try:
obj = random.choice(objgraph.by_type(line))
except Exception as e:
LOGGER.info('exception trying to objgraph a random %s: %s', line, str(e))
break
with tempfile.NamedTemporaryFile(dir='/tmp', prefix=line, suffix='.dot', mode='w') as out:
try:
objgraph.show_chain(objgraph.find_backref_chain(obj, objgraph.is_proper_module), output=out)
LOGGER.info('object %s file %s', line, out.name)
except Exception as e:
LOGGER.info('exception trying to show_chain a random %s: %s', line, str(e))
try:
os.remove(f)
except Exception as e:
LOGGER.info('exception %s removing memory_crawler file %s', str(e), f)
def check_memory():
logger.debug('Checking memory.')
logger.debug(objgraph.by_type('SpiderTask'))
logger.debug(objgraph.by_type('TaskExecutor'))
logger.debug(objgraph.by_type('Future'))
logger.debug(objgraph.by_type('PeriodicCallback'))
logger.debug(objgraph.by_type('Workspace'))
logger.debug(objgraph.by_type('MultipartRequestBodyProducer'))
logger.debug(objgraph.by_type('HTTPRequest'))
future_objs = objgraph.by_type('Future')
if future_objs:
objgraph.show_chain(
objgraph.find_backref_chain(future_objs[-1], objgraph.is_proper_module),
filename='chain.png'
)
all_objects = muppy.get_objects()
sum1 = summary.summarize(all_objects)
# Prints out a summary of the large objects
summary.print_(sum1)
_cache.remove(o)
if __name__ == '__main__':
print(type(func_to_leak))
print('counter OBJ:', objgraph.count('OBJ'))
print('counter func_to_leak:', objgraph.count('function'))
objgraph.show_growth()
try:
func_to_leak()
except:
pass
print('after call func_to_leak')
# 统计该类型对象的数目
print('counter OBJ:', objgraph.count('OBJ'))
print('counter func_to_leak:', objgraph.count('function'))
# 更具对象类型返回对象列表
print('999999999', objgraph.by_type('str'))
# 打印实例化做多的前10个对象
print(objgraph.show_most_common_types(10))
# 统计自上次调用以来增加得最多的对象，这个函数非常有利于发现潜在的内存泄露
objgraph.show_growth()
# 生产一张有关objs的引用图，看出看出对象为什么不释放，后面会利用这个API来查内存泄露
objgraph.show_backrefs(objgraph.by_type('OBJ')[0],
max_depth=10,
filename='obj.png')
# 找到一条指向obj对象的最短路径，且路径的头部节点需要满足predicate函数 （返回值为True）
objgraph.show_chain(objgraph.find_backref_chain(
objgraph.by_type('OBJ')[0], objgraph.is_proper_module),
filename='short_chain.png')
def show_chain(ob):
objgraph.show_chain(
objgraph.find_backref_chain(ob, objgraph.is_proper_module))
import objgraph as objg
from ogpath import ogpath
class MyBigFatObject(object):
pass
def comps(_cache={}):
_cache[42] = dict(foo=MyBigFatObject(), bar=MyBigFatObject())
x = MyBigFatObject()
objg.show_growth()
comps()
objg.show_growth()
print 1, objg.by_type('MyBigFatObject')
import random
objg.show_chain(objg.find_backref_chain(
random.choice(objg.by_type('MyBigFatObject')), objg.is_proper_module),
filename=ogpath('chain.png'))
def showBackrefs(self):
objType = str(self.inputWidget.text())
with self.showTempImage() as fn:
objgraph.show_chain(objgraph.find_backref_chain(objgraph.by_type(objType)[0],
objgraph.is_proper_module),
filename=fn)
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#===============================================================================
#============= enthought library imports =======================
#============= standard library imports ========================
#============= local library imports ==========================
import os
path = os.environ['PATH']
os.environ['PATH'] = '{}:/usr/local/bin'.format(path)
import objgraph
import random
import inspect
d = dict(a=1, b='2', c=3.0)
objgraph.show_chain(
objgraph.find_backref_chain(
random.choice(objgraph.by_type('dict')),
inspect.ismodule),
filename='chain.png')
#============= EOF =============================================
# Notes on debugging memory leaks in the worker
# apt-get update && apt-get install graphviz
# pip install objgraph
import objgraph
import gc
import random
from aleph.logic.documents import process_documents
objgraph.show_most_common_types()
process_documents(21)
# objgraph.show_most_common_types()
objgraph.show_chain(objgraph.find_backref_chain(random.choice(objgraph.by_type('_DebugQueryTuple')), objgraph.is_proper_module), filename='chain.png') # noqa
gc.collect()
def show_chain(i, obj):
objgraph.show_chain(
objgraph.find_backref_chain(obj, inspect.ismodule),
filename='chain.png')
def show_chain(i, obj):
objgraph.show_chain(objgraph.find_backref_chain(obj, inspect.ismodule),
filename='chain.png')
def objgraph_creategraph_backrefchain(self, args):
"""
This function can be used to display what is referencing an object to find why
an object is not garbage collected. The output is a graph.
Requirements: objgraph and graphviz installed
Fuglu has to be running as a daemon.
"fuglu_control" is used to communicate with the fuglu instance.
Examples:
$ fuglu_control objgraph_creategraph_backrefchain '{"typelist": ["SMTPServer"]}'
---------------------------------
Create Graph for backref chain:
---------------------------------
params:
* typelist: SMTPServer
* selector: random
* filename: /tmp/SMTPServer.png
Graph for one object of type SMTPServer written to /tmp/SMTPServer.png
SMTPServer.png:
#-----------------#
# module __main__ #
#-----------------#
|
| __dict__
|
#---------------#
# dict 61 items #
#---------------#
|
| controller
|
#--------------------------------------------------#
# MainController <fuglu.core.MainController objet> #
#--------------------------------------------------#
|
| __dict__
|
#---------------#
# dict 18 items #
#---------------#
|
| servers
|
#--------------#
# list 4 items #
#--------------#
|
|
|
#--------------------------------------------------------#
# SMTPServer <fuglu.connectors.smtpconnector.SMTPServer> #
#--------------------------------------------------------#
"""
res = u"---------------------------------\n" \
+ u"Create Graph for backref chain:\n" \
+ u"---------------------------------\n\n"
if OBJGRAPH_EXTENSION_ENABLED:
defaults = {
"max_depth": 20,
"filename": "",
"selector": "random",
"maxobjects": 1,
"typelist": ["Suspect"],
}
if not args:
args = {}
# fill filter lists and other vars from dict
res, inputdict = ControlSession.prepare_objectgraph_list_from_dict(
args, res, defaults)
in_chains = None
chains = None
backrefchain_ids = None
all_object_ids = None
try:
if not len(inputdict["typelist"]
) > 0 or not inputdict["typelist"]:
res += "Please define at least one type in 'typelist'"
return res
if not inputdict["filename"]:
res += "Please define a non-empty filename in 'filename' key of input dict or don't define key"
return res
if inputdict["selector"] not in [
"first", "last", "random", "all"
]:
res += 'Valid choices for selector are : "first", "last", "random", "all"'
return res
chains = []
all_object_ids = []
for otype in inputdict["typelist"]:
olist = objgraph.by_type(otype)
if not olist:
res += u"%s: no objects\n" % otype
else:
if inputdict["selector"] == "all":
examine_objs = olist
elif inputdict["selector"] == "first":
examine_objs = olist[:inputdict["maxobjects"]]
elif inputdict["selector"] == "last":
examine_objs = olist[-inputdict["maxobjects"]:]
else:
import random
if inputdict["maxobjects"] <= 1:
examine_objs = [random.choice(olist)]
else:
examine_objs = random.sample(
olist,
min(inputdict["maxobjects"], len(olist)))
if not examine_objs:
res += 'WARNING: Object to examine is None for %s' % otype
continue
for obj in examine_objs:
backrefchain = objgraph.find_backref_chain(
obj, objgraph.is_proper_module)
if not backrefchain:
res += 'WARNING: Backref chain is empty for %s' % otype
continue
chains.append(backrefchain)
backrefchain_ids = [
id(x) for x in backrefchain if x
]
all_object_ids.extend(backrefchain_ids)
self.logger.debug(
"chain is: %s" %
",".join([str(i) for i in backrefchain_ids]))
del backrefchain
del examine_objs
chains = [chain for chain in chains
if chain] # remove empty ones
class TmpObj(object):
def __init__(self, ids, logger):
self.ids = set(ids)
self.logger = logger
self.logger.debug("allids: %s" %
",".join([str(i) for i in self.ids]))
def __call__(self, x):
out = id(x) in self.ids
return out
in_chains = TmpObj(all_object_ids, self.logger)
max_depth = max(map(len, chains)) - 1
objgraph.show_backrefs([chain[-1] for chain in chains],
max_depth=max_depth,
filter=in_chains,
filename=inputdict["filename"])
res += 'Graph for one object of type(s) %s written to %s' % (
",".join(inputdict["typelist"]), inputdict['filename'])
except Exception as e:
self.logger.exception(e)
res += force_uString(e)
finally:
if in_chains:
del in_chains
if chains:
del chains
if backrefchain_ids:
del backrefchain_ids
if all_object_ids:
del all_object_ids
else:
res = u"please install module 'objgraph' and 'graphviz'"
return res
import objgraph
class Dict(dict):
def __init__(self, args={}):
dict.__init__(self, args)
class List(list):
def __init__(self, args=()):
list.__init__(self, args)
class MyClass:
def __init__(self):
self.a = []
d1 = Dict({1: 1})
d2 = Dict({2: 2})
l = List((1, 2, 3))
self.a.append(d1)
self.a.append(d2)
c = MyClass()
print 'objgraph.by_type:', objgraph.by_type('Dict')
chain = objgraph.find_backref_chain(
objgraph.by_type('Dict')[-1], inspect.ismodule)
objgraph.show_chain(chain, filename='chain.png')
# Notes on debugging memory leaks in the worker
# apt-get update && apt-get install graphviz
# pip install objgraph
import objgraph
import gc
import random
from aleph.logic.documents import process_documents
objgraph.show_most_common_types()
process_documents(21)
# objgraph.show_most_common_types()
objgraph.show_chain(
objgraph.find_backref_chain(
random.choice(objgraph.by_type("_DebugQueryTuple")),
objgraph.is_proper_module),
filename="chain.png",
) # noqa
gc.collect()