def log_mem_usage(signum, frame, fname=None):
    global _count
    _count += 1
    gc.collect()
    if not fname:
        fname = filename + '_memory_%02d.log' % _count
    with open(fname, 'wb') as f:
        f.write('gc.garbage: %d\n\n' % len(gc.garbage))
        objgraph.show_most_common_types(limit=50, file=f)
        f.write('\n\n')
        buf = StringIO()
        objgraph.show_growth(limit=50, file=buf)
        buf = buf.getvalue()
        f.write(buf)
    if _count < 2:
        return
    for tn, l in enumerate(buf.splitlines()[:10]):
        l = l.strip()
        if not l:
            continue
        type_ = l.split()[0]
        objects = objgraph.by_type(type_)
        objects = random.sample(objects, min(50, len(objects)))
        objgraph.show_chain(
            objgraph.find_backref_chain(
                objects[0],
                objgraph.is_proper_module),
            filename=fname[:-4] + '_type_%02d_backref.png' % tn
        )
        objgraph.show_backrefs(
            objects,
            max_depth=5,
            extra_info=lambda x: hex(id(x)),
            filename=fname[:-4] + '_type_%02d_backrefs.png' % tn,
        )
Example #2
0
File: job.py Project: nfredrik/mrq
  def trace_memory_stop(self):
    """ Stops measuring memory consumption """

    objgraph.show_growth(limit=10)

    trace_type = get_current_config()["trace_memory_type"]
    if trace_type:

      objgraph.show_chain(
        objgraph.find_backref_chain(
          random.choice(objgraph.by_type(trace_type)),
          objgraph.is_proper_module
        ),
        filename='%s/%s-%s.png' % (get_current_config()["trace_memory_output_dir"], trace_type, self.id)
      )

    gc.collect()
    self._memory_stop = self.worker.get_memory()

    diff = self._memory_stop - self._memory_start

    log.debug("Memory diff for job %s : %s" % (self.id, diff))

    # We need to update it later than the results, we need them off memory already.
    self.collection.update({
      "_id": self.id
    }, {"$set": {
      "memory_diff": diff
    }}, w=1)
def get_counts(prefix=""):
    test_types = [
        "Var",
        "Piecewise",
        "ScenarioTree",
        "ScenarioTreeNode",
        "Scenario",
        "_SetContainer",
        "_ConstraintArray",
    ]

    # use objgraph to check if these pyomo objects still exist
    objects = {}
    test_counts = {}
    for name in test_types:
        objects[name] = objgraph.by_type(name)
        test_counts[name] = len(objects[name])

    if True:
        for name in test_types:
            if test_counts[name] == 0:
                continue
            else:
                obj = objects[name][0]
            fname = prefix + "-" + "objgraph-{}-".format(name)
            objgraph.show_refs([obj],
                               filename=fname + "refs.png")  # too_many=50,
            objgraph.show_backrefs([obj], filename=fname + "backref.png")
            objgraph.show_chain(
                objgraph.find_backref_chain(obj, inspect.ismodule),
                filename=fname + "chain.png",
            )
    return test_counts
Example #4
0
def show_backref_from_coro():
    import random
    objgraph.show_chain(
        objgraph.find_backref_chain(
            random.choice(objgraph.by_type('coroutine')),
            objgraph.is_proper_module),
        filename='chain-to-leaked.png')
Example #5
0
def get_counts(prefix=''):
    test_types = [
        'Var', 
        'Piecewise',
        'ScenarioTree',
        'ScenarioTreeNode',
        'Scenario',
        '_SetContainer',
        '_ConstraintArray',
        ]
    
    # use objgraph to check if these pyomo objects still exist 
    objects = {} 
    test_counts = {}
    for name in test_types:
        objects[name] = objgraph.by_type( name )
        test_counts[name] = len(objects[name])
    
    if True:
        for name in test_types:
            if test_counts[name]==0: continue
            else: obj = objects[name][0]
            fname = prefix+'-'+'objgraph-{}-'.format(name)
            objgraph.show_refs( [obj], filename=fname+'refs.png') # too_many=50,
            objgraph.show_backrefs([obj], filename=fname+'backref.png')
            objgraph.show_chain(
                objgraph.find_backref_chain( obj, inspect.ismodule),
                filename=fname+'chain.png'
                )
    return test_counts    
Example #6
0
def log_mem_usage(signum, frame, fname=None):
    global _count
    _count += 1
    gc.collect()
    if not fname:
        fname = filename + '_memory_%02d.log' % _count
    with open(fname, 'wb') as f:
        f.write('gc.garbage: %d\n\n' % len(gc.garbage))
        objgraph.show_most_common_types(limit=50, file=f)
        f.write('\n\n')
        buf = StringIO()
        objgraph.show_growth(limit=50, file=buf)
        buf = buf.getvalue()
        f.write(buf)
    if _count < 2:
        return
    for tn, l in enumerate(buf.splitlines()[:10]):
        l = l.strip()
        if not l:
            continue
        type_ = l.split()[0]
        objects = objgraph.by_type(type_)
        objects = random.sample(objects, min(50, len(objects)))
        objgraph.show_chain(
            objgraph.find_backref_chain(objects[0], objgraph.is_proper_module),
            filename=fname[:-4] + '_type_%02d_backref.png' % tn)
        objgraph.show_backrefs(
            objects,
            max_depth=5,
            extra_info=lambda x: hex(id(x)),
            filename=fname[:-4] + '_type_%02d_backrefs.png' % tn,
        )
Example #7
0
def print_objects(f):
    gc.collect()
    with open(f, 'r') as fd:
        for line in fd:
            line = line.strip()
            try:
                obj = random.choice(objgraph.by_type(line))
            except Exception as e:
                LOGGER.info('exception trying to objgraph a random %s: %s',
                            line, str(e))
                break
            with tempfile.NamedTemporaryFile(dir='/tmp',
                                             prefix=line,
                                             suffix='.dot',
                                             mode='w') as out:
                try:
                    objgraph.show_chain(objgraph.find_backref_chain(
                        obj, objgraph.is_proper_module),
                                        output=out)
                    LOGGER.info('object %s file %s', line, out.name)
                except Exception as e:
                    LOGGER.info(
                        'exception trying to show_chain a random %s: %s', line,
                        str(e))
    try:
        os.remove(f)
    except Exception as e:
        LOGGER.info('exception %s removing memory_crawler file %s', str(e), f)
Example #8
0
    def trace_memory_stop(self):
        """ Stops measuring memory consumption """

        self.trace_memory_clean_caches()

        objgraph.show_growth(limit=30)

        trace_type = context.get_current_config()["trace_memory_type"]
        if trace_type:

            filename = '%s/%s-%s.png' % (context.get_current_config(
            )["trace_memory_output_dir"], trace_type, self.id)

            chain = objgraph.find_backref_chain(
                random.choice(objgraph.by_type(trace_type)),
                objgraph.is_proper_module)
            objgraph.show_chain(chain, filename=filename)
            del filename
            del chain

        gc.collect()
        self._memory_stop = self.worker.get_memory()["total"]

        diff = self._memory_stop - self._memory_start

        context.log.debug("Memory diff for job %s : %s" % (self.id, diff))

        # We need to update it later than the results, we need them off memory
        # already.
        self.collection.update({"_id": self.id},
                               {"$set": {
                                   "memory_diff": diff
                               }},
                               w=1)
Example #9
0
    def inspect(self):
        obj = self._get_request_object()
        if obj is None:
            return redirect(url_for('.overview'))

        # Figure out attributes
        attrs = tools.get_public_attrs(obj)

        # Backrefs
        referrers = objgraph.find_backref_chain(obj, objgraph.is_proper_module,
                                                2)
        sorted_referrers = sorted([(self.get_repr(o), o) for o in referrers
                                   if id(o) != id(obj)])

        # Refs
        referrents = objgraph.find_ref_chain(obj, objgraph.is_proper_module, 2)
        sorted_referrents = sorted([(self.get_repr(o), o) for o in referrents
                                    if id(o) != id(obj)])

        return self.render('flask-admin-profiler/memory/object.html',
                           obj=obj,
                           obj_text=tools.pretty_print(obj),
                           obj_id=format_id(obj),
                           obj_type=tools.get_type(obj),
                           format_id=format_id,
                           attrs=attrs,
                           referrers=sorted_referrers,
                           referrents=sorted_referrents)
Example #10
0
def test_single_dm_instance(driver_manager, db):
    """
    Our driver manager should only be linked to one PostgresDb instance in memory.

    Subsequent drivers should share the same instance to avoid extra connections (and cache duplication)
    """

    # There's a circular reference in DriverManager structure, so old test instances aren't cleaned up by
    # the reference counter.
    gc.collect()

    # For all PostgresDb instances in memory (there may be others due to other pytests), count
    # how many are connected to our instance of DriverManager
    references = 0
    for pg_instance in objgraph.by_type('PostgresDb'):
        chain = objgraph.find_backref_chain(
            pg_instance,
            max_depth=10,
            predicate=lambda o: isinstance(o, DriverManager))
        # If the referenced DriverManager is ours
        if chain[0] is driver_manager:
            print("Length {}: {}".format(
                len(chain), ' -> '.join(c.__class__.__name__ for c in chain)))
            references += 1

    assert references == 1, "Our DriverManager should only reference one PG instance"
Example #11
0
def run_last():
    run_once("last")
    objgraph.show_chain(objgraph.find_backref_chain(
        random.choice(objgraph.by_type('BoundField')),
        objgraph.is_proper_module),
                        filename=r'c:\temp\chain.png')
    print("DONE LAST")
Example #12
0
def show_chain_of_leaked():
    # LeakedOne must be leaked for this to work
    import random
    func_that_will_leak()
    objgraph.show_chain(objgraph.find_backref_chain(
        random.choice(objgraph.by_type('LeakedOne')),
        objgraph.is_proper_module),
                        filename='chain-to-leaked.png')
Example #13
0
def show_memory_refs(name):
    try: obj=objgraph.by_type(name)[0]
    except IndexError:
        print 'no object of type',name  
        return
    objgraph.show_chain(
        objgraph.find_backref_chain(obj, inspect.ismodule),
        filename='chain-{}.png'.format(name))
def run_last():
    run_once("last")
    objgraph.show_chain(
            objgraph.find_backref_chain(
                random.choice(objgraph.by_type('BoundField')),
                objgraph.is_proper_module),
                filename=r'c:\temp\chain.png')
    print( "DONE LAST")
Example #15
0
def new_item_refmaps(collect=True,
                     types=None,
                     skip=2,
                     max_per_type=10,
                     max_total=50,
                     tags=None):
    log.debug('new_item_refmaps(%s)', collect)
    if collect:
        log.debug(' forcing garbage collection')
        gc.collect()

    pid = os.getpid()
    log.debug(' finding new_ids')
    new_ids = objgraph.get_new_ids()  # dict of sets

    if STATE['new_item_refmaps'] < skip:
        log.debug(' query #%d < %d; not analyzing yet',
                  STATE['new_item_refmaps'], skip)
        STATE['new_item_refmaps'] += 1

    else:
        done = False
        events = list()
        if types is None:
            types = new_ids.keys()
        total_count = 0
        for type in types:
            type_count = 0
            log.debug(' finding backref chains for items of type=%s', type)
            for item_id in new_ids.get(type, []):
                item = objgraph.at(item_id)
                name = _obj_name(item)
                log.debug('  finding backrefs chain for item=%s', name)
                refmap = objgraph.find_backref_chain(item,
                                                     objgraph.is_proper_module)
                refmap = [_obj_name(x) for x in refmap]
                events.append({
                    'time': _now(),
                    'pid': pid,
                    'type': type,
                    'name': name,
                    'chain': refmap
                })
                total_count += 1
                type_count += 1
                if type_count > max_per_type:
                    log.debug('reached max_per_type=%d limit', max_per_type)
                    done = True
                if total_count > max_total:
                    log.debug('reached max_total=%d limit', max_total)
                    done = True
                if done:
                    break
            if done:
                break
        if events:
            _apply_tags(tags, *events)
            return {'sourcetype': 'memleak.new_item_refmaps', 'events': events}
Example #16
0
 def show_chain(self, obj_type):
     # obj_type: Myobj_Type, type:string
     ref_chain = objgraph.find_backref_chain(
         random.choice(objgraph.by_type(obj_type)),
         objgraph.is_proper_module,
         max_depth=5)
     objgraph.show_chain(ref_chain, filename='chain.dot')
     cmd = "dot -Tpng chain.dot -o chain.png"
     commands.getstatusoutput(cmd)
Example #17
0
    def process_response(self, request, response):
        objgraph.show_growth(limit=100)

        track_class_name = getattr(settings, 'ASPARAGUS_TRACK_CLASS', None)

        if track_class_name:
            objgraph.show_chain(objgraph.find_backref_chain(
                objgraph.by_type(track_class_name)[-1],
                objgraph.is_proper_module),
                                filename='leaks.png')

        return response
Example #18
0
    def backref_graph(self):
        obj = self._get_request_object()
        if obj is None:
            return redirect(url_for('.overview'))

        depth = request.args.get('depth', type=int, default=self.MAX_DEPTH)
        if depth > self.MAX_DEPTH:
            depth = self.MAX_DEPTH

        objs = objgraph.find_backref_chain(obj, objgraph.is_proper_module,
                                           depth)
        return self._render_ref_graph(objs)
Example #19
0
def graph_randobj(line):
    import objgraph
    args = shlex.split(line)
    if len(args) > 1:
        fname = args[1]
    else:
        fname = 'chain.png'
    print 'Getting random %s object...' % args[0]
    obj = random.choice(objgraph.by_type(args[0]))
    print 'Creating chain...'
    chain = objgraph.find_backref_chain(obj, objgraph.is_proper_module)
    print 'Saving chain...'
    objgraph.show_chain(chain, filename=fname)
Example #20
0
def graph_randobj(line):
    import objgraph
    args = shlex.split(line)
    if len(args) > 1:
        fname = args[1]
    else:
        fname = 'chain.png'
    print 'Getting random %s object...' % args[0]
    obj = random.choice(objgraph.by_type(args[0]))
    print 'Creating chain...'
    chain = objgraph.find_backref_chain(obj, objgraph.is_proper_module)
    print 'Saving chain...'
    objgraph.show_chain(chain, filename=fname)
Example #21
0
def gc_debug_backtraces(magictype=None, count=1):
    sio = StringIO.StringIO()
    orig_out = sys.stdout
    try:
        sys.stdout = sio
        all_objects = list(objgraph.by_type(magictype))
        random.shuffle(all_objects)
        chains = []
        for obj in all_objects[:count]:
            chains.append(
                objgraph.find_backref_chain(obj, objgraph.is_proper_module))
        sio2 = StringIO.StringIO()
        objgraph.show_chain(*chains, output=sio2)
        logging.info("%s", sio2.getvalue())
    finally:
        sys.stdout = orig_out
    def dump_memory():
        now = time() - start
        if PROFILE_MEMORY_GRAPH_BACKREF_TYPES:
            for type_ in types:
                for sample_number in xrange(
                        PROFILE_MEMORY_GRAPH_BACKREF_AMOUNT):
                    objects = objgraph.by_type(type_)
                    if objects:
                        objgraph.show_chain(objgraph.find_backref_chain(
                            random.choice(objects), objgraph.is_proper_module),
                                            filename=objgraph_out_file %
                                            (type_, now, sample_number))
                    else:
                        logger.error("No objects of type %s found!", type_)

        scanner.dump_all_objects(meliae_out_file % now)
Example #23
0
def graphMemProfile(func, args, kw_args):
    import objgraph
    # objgraph.show_most_common_types()
    objgraph.show_growth()
    func(*args, **kw_args)
    objgraph.show_growth()
    import random
    str(
        objgraph.show_chain(objgraph.find_backref_chain(
            random.choice(objgraph.by_type('staticmethod')),
            objgraph.is_proper_module),
                            filename='chain.png'))
    #objgraph.show_refs([func(*args,**kw_args)], filename='refs.png')
    #objgraph.show_backrefs([func(*args,**kw_args)], filename='backrefs.png')

    #import pdb; pdb.set_trace()
    objgraph.show_growth()
def backref_objects_by_type(obj_type):
    """Find all of the objects by a given type and graph back-reference sample.

    Randomly graph the back-references for a small number of instances.
    """
    if LOG.isEnabledFor(logging.DEBUG3) and _ObjGraph:
        #
        # Any nested function will hold onto some local state and leak.
        # Since gc.get_objects() contains the very frame running this code,
        # you get a circular reference.
        #
        gc.collect()
        all_objects = gc.get_objects()
        try:
            objects = objgraph.by_type(obj_type, all_objects)
        finally:
            del all_objects
        LOG.debug3("{} objects of type {} on heap".format(
            len(objects), obj_type))
        if objects:
            try:
                obj = random.choice(objects)
            finally:
                del objects
            # prune the unhashables from the garbage list to avoid errors with set()
            garbage = [o for o in gc.garbage if is_hashable(o)]
            try:
                ### this seems to interfere with the measurements...
                chain = objgraph.find_backref_chain(obj,
                                                    inspect.ismodule,
                                                    extra_ignore=garbage)
            finally:
                del garbage
                # Delete after calling objgraph, which invokes gc.collect()
                del gc.garbage[:]
            try:
                LOG.debug3("{} chain for {}".format(obj_type, obj))
                for link in chain:
                    LOG.debug3("chain link: {}: {}".format(
                        type(link),
                        str(link)[:240]))
            finally:
                del chain
Example #25
0
def print_objects(f):
    gc.collect()
    with open(f, 'r') as fd:
        for line in fd:
            line = line.strip()
            try:
                obj = random.choice(objgraph.by_type(line))
            except Exception as e:
                LOGGER.info('exception trying to objgraph a random %s: %s', line, str(e))
                break
            with tempfile.NamedTemporaryFile(dir='/tmp', prefix=line, suffix='.dot', mode='w') as out:
                try:
                    objgraph.show_chain(objgraph.find_backref_chain(obj, objgraph.is_proper_module), output=out)
                    LOGGER.info('object %s file %s', line, out.name)
                except Exception as e:
                    LOGGER.info('exception trying to show_chain a random %s: %s', line, str(e))
    try:
        os.remove(f)
    except Exception as e:
        LOGGER.info('exception %s removing memory_crawler file %s', str(e), f)
Example #26
0
    def check_memory():
        logger.debug('Checking memory.')
        logger.debug(objgraph.by_type('SpiderTask'))
        logger.debug(objgraph.by_type('TaskExecutor'))
        logger.debug(objgraph.by_type('Future'))
        logger.debug(objgraph.by_type('PeriodicCallback'))
        logger.debug(objgraph.by_type('Workspace'))
        logger.debug(objgraph.by_type('MultipartRequestBodyProducer'))
        logger.debug(objgraph.by_type('HTTPRequest'))
        future_objs = objgraph.by_type('Future')
        if future_objs:
            objgraph.show_chain(
                objgraph.find_backref_chain(future_objs[-1], objgraph.is_proper_module),
                filename='chain.png'
            )

        all_objects = muppy.get_objects()
        sum1 = summary.summarize(all_objects)
        # Prints out a summary of the large objects
        summary.print_(sum1)
Example #27
0
    _cache.remove(o)


if __name__ == '__main__':
    print(type(func_to_leak))
    print('counter OBJ:', objgraph.count('OBJ'))
    print('counter func_to_leak:', objgraph.count('function'))
    objgraph.show_growth()
    try:
        func_to_leak()
    except:
        pass
    print('after call func_to_leak')
    # 统计该类型对象的数目
    print('counter OBJ:', objgraph.count('OBJ'))
    print('counter func_to_leak:', objgraph.count('function'))
    # 更具对象类型返回对象列表
    print('999999999', objgraph.by_type('str'))
    # 打印实例化做多的前10个对象
    print(objgraph.show_most_common_types(10))
    # 统计自上次调用以来增加得最多的对象,这个函数非常有利于发现潜在的内存泄露
    objgraph.show_growth()
    # 生产一张有关objs的引用图,看出看出对象为什么不释放,后面会利用这个API来查内存泄露
    objgraph.show_backrefs(objgraph.by_type('OBJ')[0],
                           max_depth=10,
                           filename='obj.png')
    # 找到一条指向obj对象的最短路径,且路径的头部节点需要满足predicate函数 (返回值为True)
    objgraph.show_chain(objgraph.find_backref_chain(
        objgraph.by_type('OBJ')[0], objgraph.is_proper_module),
                        filename='short_chain.png')
Example #28
0
def show_chain(ob):
    objgraph.show_chain(
        objgraph.find_backref_chain(ob, objgraph.is_proper_module))
Example #29
0
import objgraph as objg
from ogpath import ogpath


class MyBigFatObject(object):
    pass


def comps(_cache={}):
    _cache[42] = dict(foo=MyBigFatObject(), bar=MyBigFatObject())

    x = MyBigFatObject()


objg.show_growth()
comps()
objg.show_growth()

print 1, objg.by_type('MyBigFatObject')

import random
objg.show_chain(objg.find_backref_chain(
    random.choice(objg.by_type('MyBigFatObject')), objg.is_proper_module),
                filename=ogpath('chain.png'))
Example #30
0
 def showBackrefs(self):
     objType = str(self.inputWidget.text())
     with self.showTempImage() as fn:
         objgraph.show_chain(objgraph.find_backref_chain(objgraph.by_type(objType)[0],
                                                         objgraph.is_proper_module),
                             filename=fn)
Example #31
0
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#===============================================================================

#============= enthought library imports =======================
#============= standard library imports ========================
#============= local library imports  ==========================
import os

path = os.environ['PATH']
os.environ['PATH'] = '{}:/usr/local/bin'.format(path)

import objgraph
import random
import inspect

d = dict(a=1, b='2', c=3.0)
objgraph.show_chain(
                    objgraph.find_backref_chain(
                                                random.choice(objgraph.by_type('dict')),
                                                inspect.ismodule),
                    filename='chain.png')


#============= EOF =============================================
Example #32
0
# Notes on debugging memory leaks in the worker

# apt-get update && apt-get install graphviz
# pip install objgraph

import objgraph
import gc
import random
from aleph.logic.documents import process_documents
objgraph.show_most_common_types()

process_documents(21)
# objgraph.show_most_common_types()

objgraph.show_chain(objgraph.find_backref_chain(random.choice(objgraph.by_type('_DebugQueryTuple')), objgraph.is_proper_module), filename='chain.png')  # noqa
gc.collect()
Example #33
0
def show_chain(i, obj):
    objgraph.show_chain(
     objgraph.find_backref_chain(obj, inspect.ismodule),
     filename='chain.png')
Example #34
0
def show_chain(i, obj):
    objgraph.show_chain(objgraph.find_backref_chain(obj, inspect.ismodule),
                        filename='chain.png')
Example #35
0
    def objgraph_creategraph_backrefchain(self, args):
        """
        This function can be used to display what is referencing an object to find why
        an object is not garbage collected. The output is a graph.

        Requirements: objgraph and graphviz installed

        Fuglu has to be running as a daemon.
        "fuglu_control" is used to communicate with the fuglu instance.

        Examples:

            $ fuglu_control objgraph_creategraph_backrefchain '{"typelist": ["SMTPServer"]}'

            ---------------------------------
            Create Graph for backref chain:
            ---------------------------------

            params:
            * typelist: SMTPServer
            * selector: random
            * filename: /tmp/SMTPServer.png

            Graph for one object of type SMTPServer written to /tmp/SMTPServer.png

            SMTPServer.png:

            #-----------------#
            # module __main__ #
            #-----------------#
                   |
                   |   __dict__
                   |
            #---------------#
            # dict 61 items #
            #---------------#
                   |
                   |   controller
                   |
            #--------------------------------------------------#
            # MainController <fuglu.core.MainController objet> #
            #--------------------------------------------------#
                   |
                   |   __dict__
                   |
            #---------------#
            # dict 18 items #
            #---------------#
                   |
                   |   servers
                   |
            #--------------#
            # list 4 items #
            #--------------#
                   |
                   |
                   |
            #--------------------------------------------------------#
            # SMTPServer <fuglu.connectors.smtpconnector.SMTPServer> #
            #--------------------------------------------------------#

        """
        res = u"---------------------------------\n" \
              + u"Create Graph for backref chain:\n" \
              + u"---------------------------------\n\n"

        if OBJGRAPH_EXTENSION_ENABLED:
            defaults = {
                "max_depth": 20,
                "filename": "",
                "selector": "random",
                "maxobjects": 1,
                "typelist": ["Suspect"],
            }

            if not args:
                args = {}

            # fill filter lists and other vars from dict
            res, inputdict = ControlSession.prepare_objectgraph_list_from_dict(
                args, res, defaults)
            in_chains = None
            chains = None
            backrefchain_ids = None
            all_object_ids = None

            try:
                if not len(inputdict["typelist"]
                           ) > 0 or not inputdict["typelist"]:
                    res += "Please define at least one type in 'typelist'"
                    return res

                if not inputdict["filename"]:
                    res += "Please define a non-empty filename in 'filename' key of input dict or don't define key"
                    return res

                if inputdict["selector"] not in [
                        "first", "last", "random", "all"
                ]:
                    res += 'Valid choices for selector are : "first", "last", "random", "all"'
                    return res

                chains = []
                all_object_ids = []
                for otype in inputdict["typelist"]:
                    olist = objgraph.by_type(otype)
                    if not olist:
                        res += u"%s: no objects\n" % otype
                    else:
                        if inputdict["selector"] == "all":
                            examine_objs = olist
                        elif inputdict["selector"] == "first":
                            examine_objs = olist[:inputdict["maxobjects"]]
                        elif inputdict["selector"] == "last":
                            examine_objs = olist[-inputdict["maxobjects"]:]
                        else:
                            import random
                            if inputdict["maxobjects"] <= 1:
                                examine_objs = [random.choice(olist)]
                            else:
                                examine_objs = random.sample(
                                    olist,
                                    min(inputdict["maxobjects"], len(olist)))

                        if not examine_objs:
                            res += 'WARNING: Object to examine is None for %s' % otype
                            continue

                        for obj in examine_objs:
                            backrefchain = objgraph.find_backref_chain(
                                obj, objgraph.is_proper_module)

                            if not backrefchain:
                                res += 'WARNING: Backref chain is empty for %s' % otype
                                continue

                            chains.append(backrefchain)
                            backrefchain_ids = [
                                id(x) for x in backrefchain if x
                            ]
                            all_object_ids.extend(backrefchain_ids)
                            self.logger.debug(
                                "chain is: %s" %
                                ",".join([str(i) for i in backrefchain_ids]))
                            del backrefchain
                        del examine_objs

                chains = [chain for chain in chains
                          if chain]  # remove empty ones

                class TmpObj(object):
                    def __init__(self, ids, logger):
                        self.ids = set(ids)
                        self.logger = logger
                        self.logger.debug("allids: %s" %
                                          ",".join([str(i) for i in self.ids]))

                    def __call__(self, x):
                        out = id(x) in self.ids
                        return out

                in_chains = TmpObj(all_object_ids, self.logger)

                max_depth = max(map(len, chains)) - 1
                objgraph.show_backrefs([chain[-1] for chain in chains],
                                       max_depth=max_depth,
                                       filter=in_chains,
                                       filename=inputdict["filename"])

                res += 'Graph for one object of type(s) %s written to %s' % (
                    ",".join(inputdict["typelist"]), inputdict['filename'])

            except Exception as e:
                self.logger.exception(e)
                res += force_uString(e)
            finally:
                if in_chains:
                    del in_chains
                if chains:
                    del chains
                if backrefchain_ids:
                    del backrefchain_ids
                if all_object_ids:
                    del all_object_ids
        else:
            res = u"please install module 'objgraph' and 'graphviz'"
        return res
Example #36
0
import objgraph


class Dict(dict):
    def __init__(self, args={}):
        dict.__init__(self, args)


class List(list):
    def __init__(self, args=()):
        list.__init__(self, args)


class MyClass:
    def __init__(self):
        self.a = []
        d1 = Dict({1: 1})
        d2 = Dict({2: 2})
        l = List((1, 2, 3))
        self.a.append(d1)
        self.a.append(d2)


c = MyClass()

print 'objgraph.by_type:', objgraph.by_type('Dict')
chain = objgraph.find_backref_chain(
    objgraph.by_type('Dict')[-1], inspect.ismodule)
objgraph.show_chain(chain, filename='chain.png')
Example #37
0
# Notes on debugging memory leaks in the worker

# apt-get update && apt-get install graphviz
# pip install objgraph

import objgraph
import gc
import random
from aleph.logic.documents import process_documents

objgraph.show_most_common_types()

process_documents(21)
# objgraph.show_most_common_types()

objgraph.show_chain(
    objgraph.find_backref_chain(
        random.choice(objgraph.by_type("_DebugQueryTuple")),
        objgraph.is_proper_module),
    filename="chain.png",
)  # noqa
gc.collect()