コード例 #1
0
ファイル: test_subscriptions.py プロジェクト: prezi/dagster
def test_compute_log_subscription(mock_watch_completed):
    mock_watch_completed.return_value = False

    schema = create_schema()
    server = DagsterSubscriptionServer(schema=schema)

    with instance_for_test() as instance:
        run = execute_pipeline(example_pipeline, instance=instance)
        assert run.success
        assert run.run_id

        with create_subscription_context(instance) as context:
            start_subscription(
                server,
                context,
                COMPUTE_LOG_SUBSCRIPTION,
                {
                    "runId": run.run_id,
                    "stepKey": "example_solid",
                    "ioType": "STDERR",
                },
            )
            gc.collect()
            assert len(objgraph.by_type("SubscriptionObserver")) == 1
            assert len(objgraph.by_type("ComputeLogSubscription")) == 1
            end_subscription(server, context)
            gc.collect()
            assert len(objgraph.by_type("SubscriptionObserver")) == 0
            assert len(objgraph.by_type("ComputeLogSubscription")) == 0
コード例 #2
0
ファイル: cli.py プロジェクト: abathur/anaphora
def _objgraphs():
	import objgraph
	print(objgraph.show_backrefs(objgraph.by_type("requirement"), filename='/home/anaphora/backrefs.png'))
	print(objgraph.show_backrefs(objgraph.by_type("Stat"), filename='/home/anaphora/statrefs.png'))
	print(objgraph.show_backrefs(objgraph.by_type("QueryAPI"), filename='/home/anaphora/dbrefs.png'))
	print(objgraph.show_backrefs(objgraph.by_type("Noun"), filename='/home/anaphora/nounrefs.png'))
	print(objgraph.show_refs(objgraph.by_type("Noun"), filename='/home/anaphora/nounrefs2.png'))
コード例 #3
0
ファイル: test_app.py プロジェクト: prezi/dagster
def test_graphql_ws_success(instance, empty_app):
    @pipeline
    def _test():
        pass

    result = execute_pipeline(_test, instance=instance)
    run_id = result.run_id
    # wtf pylint
    # pylint: disable=not-context-manager
    with TestClient(empty_app).websocket_connect("/graphql", GraphQLWS.PROTOCOL) as ws:
        ws.send_json({"type": GraphQLWS.CONNECTION_INIT})
        ws.send_json(
            {
                "type": GraphQLWS.START,
                "id": "1",
                "payload": {"query": EVENT_LOG_SUBSCRIPTION, "variables": {"runId": run_id}},
            }
        )

        response = ws.receive_json()
        assert response["type"] == GraphQLWS.CONNECTION_ACK

        response = ws.receive_json()
        assert response["id"] == "1"

        assert response["type"] == GraphQLWS.DATA

        gc.collect()
        assert len(objgraph.by_type("PipelineRunObservableSubscribe")) == 1

    # after exiting the context manager and closing the connection
    gc.collect()
    assert len(objgraph.by_type("PipelineRunObservableSubscribe")) == 0
コード例 #4
0
ファイル: conftest.py プロジェクト: sunn-e/glue
def pytest_unconfigure(config):

    os.environ.pop('GLUE_TESTING')

    # Reset configuration directory to original one
    from glue import config
    config.CFG_DIR = CFG_DIR_ORIG

    # Remove reference to QApplication to prevent segmentation fault on PySide
    try:
        from glue.utils.qt import app
        app.qapp = None
    except Exception:  # for when we run the tests without the qt directories
        # Note that we catch any exception, not just ImportError, because
        # QtPy can raise a PythonQtError.
        pass

    if OBJGRAPH_INSTALLED and not ON_APPVEYOR:

        # Make sure there are no lingering references to GlueApplication
        obj = objgraph.by_type('GlueApplication')
        if len(obj) > 0:
            objgraph.show_backrefs(objgraph.by_type('GlueApplication'))
            warnings.warn(
                'There are {0} remaining references to GlueApplication'.format(
                    len(obj)))
コード例 #5
0
    def test_dispose(self, server, load_stubs, spec_dir, grpc_port, protobufs):
        """ Regression test for client connection part of
        https://github.com/nameko/nameko-grpc/issues/40
        """
        stubs = load_stubs("example")

        clients = {}
        for _ in range(self.COUNT):
            client = Client(
                "//localhost:{}".format(grpc_port),
                stubs.exampleStub,
                "none",
                "high",
                False,
            )
            proxy = client.start()
            clients[client] = proxy

        gc.collect()
        assert len(objgraph.by_type("ClientConnectionManager")) == self.COUNT

        for client in clients.keys():
            client.stop()

        gc.collect()
        assert len(objgraph.by_type("ClientConnectionManager")) == 0
コード例 #6
0
def test_event_log_subscription_chunked():
    schema = create_schema()
    server = DagsterSubscriptionServer(schema=schema)

    with instance_for_test() as instance, environ(
        {"DAGIT_EVENT_LOAD_CHUNK_SIZE": "2"}):
        run = execute_pipeline(example_pipeline, instance=instance)
        assert run.success
        assert run.run_id

        with create_subscription_context(instance) as context:
            start_subscription(server, context, EVENT_LOG_SUBSCRIPTION,
                               {"runId": run.run_id})
            gc.collect()
            assert len(objgraph.by_type("SubscriptionObserver")) == 1
            subs = objgraph.by_type("PipelineRunObservableSubscribe")
            assert len(subs) == 1
            subscription_obj = subs[0]
            end_subscription(server, context)
            subscription_obj.stopped.wait(30)
            subs = None
            subscription_obj = None
            gc.collect()

            assert len(objgraph.by_type("SubscriptionObserver")) == 0
            assert len(objgraph.by_type("PipelineRunObservableSubscribe")) == 0
コード例 #7
0
def test_compute_log_subscription(mock_watch_completed):
    mock_watch_completed.return_value = False

    with instance_for_test() as instance:
        run = execute_pipeline(example_pipeline, instance=instance)
        assert run.success
        assert run.run_id

        with create_asgi_client(instance) as client:
            # pylint: disable=not-context-manager
            with client.websocket_connect("/graphql", GraphQLWS.PROTOCOL) as ws:

                start_subscription(
                    ws,
                    COMPUTE_LOG_SUBSCRIPTION,
                    {
                        "runId": run.run_id,
                        "stepKey": "example_solid",
                        "ioType": "STDERR",
                    },
                )
                gc.collect()
                assert len(objgraph.by_type("ComputeLogSubscription")) == 1
                end_subscription(ws)

            gc.collect()
            assert len(objgraph.by_type("ComputeLogSubscription")) == 0
コード例 #8
0
    def test_dispose(self, server, load_stubs, spec_dir, grpc_port, protobufs):
        """ Regression test for server connection part of
        https://github.com/nameko/nameko-grpc/issues/40
        """
        stubs = load_stubs("example")

        clients = {}
        for _ in range(self.COUNT):
            client = Client(
                "//localhost:{}".format(grpc_port),
                stubs.exampleStub,
                "none",
                "high",
                False,
            )
            proxy = client.start()
            clients[client] = proxy

        gc.collect()
        assert len(objgraph.by_type("ServerConnectionManager")) == self.COUNT

        for client, proxy in clients.items():
            response = proxy.unary_unary(protobufs.ExampleRequest(value="A"))
            assert response.message == "A"
            client.stop()

        # while the server is running there is always exactly one server connnection
        # that remains in memory -- the previously used one remains in the closure of
        # the "while is accepting" loop inside GrpcServer.run
        gc.collect()
        assert len(objgraph.by_type("ServerConnectionManager")) == 1
コード例 #9
0
ファイル: test_subscriptions.py プロジェクト: prezi/dagster
def test_generic_subscriptions():
    server = DagsterSubscriptionServer(schema=None)
    with instance_for_test() as instance:
        with create_subscription_context(instance) as context:
            start_subscription(server, context)
            gc.collect()
            assert len(objgraph.by_type("SubscriptionObserver")) == 1
            end_subscription(server, context)
            gc.collect()
            assert len(objgraph.by_type("SubscriptionObserver")) == 0
コード例 #10
0
def memory_content(interactive=False):
    typestats = dict(objgraph.typestats())

    while True:
        types = []
        for itype, type_name in enumerate(sorted(typestats.keys())):
            print '%d %s: %d' % (itype, type_name, typestats[type_name])
            types.append(type_name)
            cls = eval(type_name)
            if hasattr(cls, '__len__'):
                for obj in objgraph.by_type(type_name):
                    if len(obj) > 1000:
                        print '   Large object (n=%d) [%s ...]' % (len(obj),
                                                                   str(obj[0]))

        if not interactive:
            break

        print 'Inspect? (0-%d or q):' % itype
        response = sys.stdin.readline().strip()
        if response == 'q':
            break

        try:
            itype = int(response)
            objs = objgraph.by_type(types[itype])
        except:
            print 'Unrecognized input %s' % response
            continue

        while True:
            print 'Item #? (0-%d or q):' % len(objs)
            response = sys.stdin.readline()
            if response == 'q':
                break

            try:
                iobj = int(response)
                obj = objs[iobj]
            except:
                print 'Unrecognized input %s' % response
                continue

            while True:
                print 'Expression? (use "obj", e.g., len(obj), or q):'
                response = sys.stdin.readline()
                if response == 'q':
                    break

                try:
                    print eval(response)
                except:
                    print 'Unrecognized input %s' % response
                    continue
コード例 #11
0
ファイル: profiling.py プロジェクト: Trulsmatias/MLProject
def obj():
    _log.debug('Generations: {}'.format(len(objgraph.by_type('Generation'))))
    _log.debug('Individuals: {}'.format(len(objgraph.by_type('Individual'))))
    _log.debug('NNAgents: {}'.format(len(objgraph.by_type('NNAgent'))))
    _log.debug('Sequentials: {}'.format(len(objgraph.by_type('Sequential'))))

    for i, obj in enumerate(objgraph.by_type('Sequential')):
        # objgraph.show_chain(objgraph.find_backref_chain(obj, objgraph.is_proper_module),
        #                    filename='refs/chain_{}.png'.format(i))
        # objgraph.show_backrefs(obj, max_depth=5, too_many=5, filename='refs/backrefs_{}.png'.format(i))
        pass
コード例 #12
0
ファイル: debug.py プロジェクト: SmartDataProjects/dynamo
def memory_content(interactive = False):
    typestats = dict(objgraph.typestats())

    while True:
        types = []
        for itype, type_name in enumerate(sorted(typestats.keys())):
            print '%d %s: %d' % (itype, type_name, typestats[type_name])
            types.append(type_name)
            cls = eval(type_name)
            if hasattr(cls, '__len__'):
                for obj in objgraph.by_type(type_name):
                    if len(obj) > 1000:
                        print '   Large object (n=%d) [%s ...]' % (len(obj), str(obj[0]))

        if not interactive:
            break

        print 'Inspect? (0-%d or q):' % itype
        response = sys.stdin.readline().strip()
        if response == 'q':
            break

        try:
            itype = int(response)
            objs = objgraph.by_type(types[itype])
        except:
            print 'Unrecognized input %s' % response
            continue

        while True:
            print 'Item #? (0-%d or q):' % len(objs)
            response = sys.stdin.readline()
            if response == 'q':
                break
    
            try:
                iobj = int(response)
                obj = objs[iobj]
            except:
                print 'Unrecognized input %s' % response
                continue

            while True:
                print 'Expression? (use "obj", e.g., len(obj), or q):'
                response = sys.stdin.readline()
                if response == 'q':
                    break
        
                try:
                    print eval(response)
                except:
                    print 'Unrecognized input %s' % response
                    continue
コード例 #13
0
def memory_tracing(key_type: str = "lineno", limit: int = 15):
    """
    Traces memory consumption and prints memory-usage statistics when leaving the context
    :param key_type:
    :param limit:
    :return:
    """
    tracemalloc.start()
    print("--- Tracing memory... ---")
    try:
        # Do computation ...
        yield None
    finally:
        snapshot = tracemalloc.take_snapshot()
        # snapshot = snapshot.filter_traces((
        #     tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
        #     tracemalloc.Filter(False, "<unknown>"),
        # ))
        top_stats = snapshot.statistics(key_type)
        print("--- Memory usage statistics: ---")
        print("Top %s lines:" % limit)
        for index, stat in enumerate(top_stats[:limit], 1):
            frame = stat.traceback[0]
            # replace "/path/to/module/file.py" with "module/file.py"
            filename = os.sep.join(frame.filename.split(os.sep)[-2:])
            print("#%s: %s:%s: %.1f KiB"
                  % (index, filename, frame.lineno, stat.size / 1024))
            line = linecache.getline(frame.filename, frame.lineno).strip()
            if line:
                print('    %s' % line)

        other = top_stats[limit:]
        if other:
            size = sum(stat.size for stat in other)
            print("%s other: %.1f KiB" % (len(other), size / 1024))
        total = sum(stat.size for stat in top_stats)
        print("\nTotal allocated size: %.1f KiB" % (total / 1024))

        # May also be useful:
        import objgraph
        print("\nTypes of most common instances:")
        objgraph.show_most_common_types(limit=limit)
        print("\nObjects that do not have any referents:")
        objgraph.get_leaking_objects()
        print("\nIncrease in peak object counts since last call:")
        objgraph.show_growth(limit=limit)
        print("\ntuple objects tracked by the garbage collector:")
        objgraph.by_type('tuple')
        print("\ndict objects tracked by the garbage collector:")
        objgraph.by_type('dict')
        print("--- End of memory tracing ---")
コード例 #14
0
ファイル: watcher.py プロジェクト: Cesareans/Flow
 def checkCycleReference():
     try:
         import objgraph
     except ImportError:
         return
     # for tp in objgraph.by_type("Entity"):
     #     filename = ProjectPath().filepath("output", 'chain_{}.png'.format(time.time()))
     #     objgraph.show_backrefs(tp, max_depth=10, filename=filename)
     logging.debug(objgraph.by_type("Entity"))
     filename = ProjectPath().filepath("output",
                                       'chain_{}.png'.format(time.time()))
     objgraph.show_refs(objgraph.by_type("Entity"),
                        max_depth=10,
                        filename=filename)
コード例 #15
0
 def save_graph(self, obj_name, max_depth=5):
     if self.enable:
         if type(obj_name) is not str:
             print('input class name into "obj_name" parameter.')
             return
         refs_filepath = os.path.join(self.output_path,
                                      obj_name + '.refs..png')
         backrefs_filepath = os.path.join(self.output_path,
                                          obj_name + '.backrefs..png')
         objgraph.show_backrefs(objgraph.by_type(obj_name),
                                max_depth=max_depth,
                                filename=backrefs_filepath)
         objgraph.show_refs(objgraph.by_type(obj_name),
                            max_depth=max_depth,
                            filename=refs_filepath)
コード例 #16
0
async def test_memory_objects(server: Server, browser_name: str) -> None:
    async with async_playwright() as p:
        browser = await p[browser_name].launch()
        page = await browser.new_page()
        await page.goto(server.EMPTY_PAGE)

        page.on("dialog", lambda dialog: dialog.dismiss())
        for _ in range(100):
            await page.evaluate("""async () => alert()""")

        await page.route("**/", lambda route, _: route.fulfill(body="OK"))

        for _ in range(100):
            response = await page.evaluate("""async () => (await fetch("/")).text()""")
            assert response == "OK"

        await browser.close()

    gc.collect()

    pw_objects: defaultdict = defaultdict(int)
    for o in objgraph.by_type("dict"):
        name = o.get("_type")
        if not name:
            continue
        pw_objects[name] += 1

    assert "Dialog" not in pw_objects
    assert "Request" not in pw_objects
    assert "Route" not in pw_objects
コード例 #17
0
def test_commits_pickling():
    repo_path = os.path.join(settings.get('git_repositories_dir'), 'trfl')
    repo = RepositoryMining(repo_path)

    commits = list(repo.traverse_commits())[:10]

    cnt_before = len(objgraph.by_type("Commit"))
    print(f'Starting with {cnt_before}')
    for n, commit in enumerate(commits):
        pickle.dumps(commit)
        print(f'#{n+1} {len(objgraph.by_type("Commit"))}')

    cnt_after = len(objgraph.by_type("Commit"))
    print(f'Ending with {cnt_after}')

    assert cnt_before == cnt_after  # Track issue on https://github.com/ishepard/pydriller/issues/102
コード例 #18
0
def log_mem_usage(signum, frame, fname=None):
    global _count
    _count += 1
    gc.collect()
    if not fname:
        fname = filename + '_memory_%02d.log' % _count
    with open(fname, 'wb') as f:
        f.write('gc.garbage: %d\n\n' % len(gc.garbage))
        objgraph.show_most_common_types(limit=50, file=f)
        f.write('\n\n')
        buf = StringIO()
        objgraph.show_growth(limit=50, file=buf)
        buf = buf.getvalue()
        f.write(buf)
    if _count < 2:
        return
    for tn, l in enumerate(buf.splitlines()[:10]):
        l = l.strip()
        if not l:
            continue
        type_ = l.split()[0]
        objects = objgraph.by_type(type_)
        objects = random.sample(objects, min(50, len(objects)))
        objgraph.show_chain(
            objgraph.find_backref_chain(objects[0], objgraph.is_proper_module),
            filename=fname[:-4] + '_type_%02d_backref.png' % tn)
        objgraph.show_backrefs(
            objects,
            max_depth=5,
            extra_info=lambda x: hex(id(x)),
            filename=fname[:-4] + '_type_%02d_backrefs.png' % tn,
        )
コード例 #19
0
def print_objects(f):
    gc.collect()
    with open(f, 'r') as fd:
        for line in fd:
            line = line.strip()
            try:
                obj = random.choice(objgraph.by_type(line))
            except Exception as e:
                LOGGER.info('exception trying to objgraph a random %s: %s',
                            line, str(e))
                break
            with tempfile.NamedTemporaryFile(dir='/tmp',
                                             prefix=line,
                                             suffix='.dot',
                                             mode='w') as out:
                try:
                    objgraph.show_chain(objgraph.find_backref_chain(
                        obj, objgraph.is_proper_module),
                                        output=out)
                    LOGGER.info('object %s file %s', line, out.name)
                except Exception as e:
                    LOGGER.info(
                        'exception trying to show_chain a random %s: %s', line,
                        str(e))
    try:
        os.remove(f)
    except Exception as e:
        LOGGER.info('exception %s removing memory_crawler file %s', str(e), f)
コード例 #20
0
def log_mem_usage(signum, frame, fname=None):
    global _count
    _count += 1
    gc.collect()
    if not fname:
        fname = filename + '_memory_%02d.log' % _count
    with open(fname, 'wb') as f:
        f.write('gc.garbage: %d\n\n' % len(gc.garbage))
        objgraph.show_most_common_types(limit=50, file=f)
        f.write('\n\n')
        buf = StringIO()
        objgraph.show_growth(limit=50, file=buf)
        buf = buf.getvalue()
        f.write(buf)
    if _count < 2:
        return
    for tn, l in enumerate(buf.splitlines()[:10]):
        l = l.strip()
        if not l:
            continue
        type_ = l.split()[0]
        objects = objgraph.by_type(type_)
        objects = random.sample(objects, min(50, len(objects)))
        objgraph.show_chain(
            objgraph.find_backref_chain(
                objects[0],
                objgraph.is_proper_module),
            filename=fname[:-4] + '_type_%02d_backref.png' % tn
        )
        objgraph.show_backrefs(
            objects,
            max_depth=5,
            extra_info=lambda x: hex(id(x)),
            filename=fname[:-4] + '_type_%02d_backrefs.png' % tn,
        )
コード例 #21
0
async def test_memory_objects() -> None:
    async with async_playwright() as p:
        browser = await p.chromium.launch()
        page = await browser.new_page()
        await page.goto("https://example.com")

        page.on("dialog", lambda dialog: dialog.dismiss())
        for _ in range(100):
            await page.evaluate("""async () => alert()""")

        await page.route("**/", lambda route, _: route.fulfill(body="OK"))

        for _ in range(100):
            response = await page.evaluate(
                """async () => (await fetch("/")).text()""")
            assert response == "OK"

        await browser.close()

    gc.collect()

    df_dicts = pd.DataFrame()
    df_dicts["dicts"] = objgraph.by_type("dict")
    df_dicts["pw_types"] = df_dicts["dicts"].apply(lambda x: x.get("_type"))

    head = df_dicts["pw_types"].value_counts().head(20)
    assert "Dialog" not in head.items()
    assert "Request" not in head.items()
    assert "Route" not in head.items()
コード例 #22
0
def get_counts(prefix=""):
    test_types = [
        "Var",
        "Piecewise",
        "ScenarioTree",
        "ScenarioTreeNode",
        "Scenario",
        "_SetContainer",
        "_ConstraintArray",
    ]

    # use objgraph to check if these pyomo objects still exist
    objects = {}
    test_counts = {}
    for name in test_types:
        objects[name] = objgraph.by_type(name)
        test_counts[name] = len(objects[name])

    if True:
        for name in test_types:
            if test_counts[name] == 0:
                continue
            else:
                obj = objects[name][0]
            fname = prefix + "-" + "objgraph-{}-".format(name)
            objgraph.show_refs([obj],
                               filename=fname + "refs.png")  # too_many=50,
            objgraph.show_backrefs([obj], filename=fname + "backref.png")
            objgraph.show_chain(
                objgraph.find_backref_chain(obj, inspect.ismodule),
                filename=fname + "chain.png",
            )
    return test_counts
コード例 #23
0
    def teardown_method(self, method):

        if self.viewer is not None:
            self.viewer.close()
        self.application.close()

        # Matplotlib 3.5 introduced a memory leak when resizing the viewer
        # in https://github.com/matplotlib/matplotlib/pull/19255 so for now
        # we skip the affected test for the objgraph testing
        if method.__name__ == 'test_aspect_resize':
            return

        # The following seems to fail on Python 3.10 - to be investigated
        if sys.version_info[:2] >= (3, 10) and method.__name__ == 'test_session_round_trip':
            return

        # The following is a check to make sure that once the viewer and
        # application have been closed, there are no leftover references to
        # the data viewer. This was introduced because there were previously
        # circular references that meant that viewer instances were not
        # properly garbage collected, which in turn meant they still reacted
        # in some cases to events.
        if OBJGRAPH_INSTALLED:
            self.viewer = None
            self.application = None
            if self.viewer_count > self.viewer_count_start:
                objgraph.show_backrefs(objgraph.by_type(self.viewer_cls.__name__))
                raise ValueError("No net viewers should be created in tests")
コード例 #24
0
def run_last():
    run_once("last")
    objgraph.show_chain(objgraph.find_backref_chain(
        random.choice(objgraph.by_type('BoundField')),
        objgraph.is_proper_module),
                        filename=r'c:\temp\chain.png')
    print("DONE LAST")
コード例 #25
0
ファイル: job.py プロジェクト: nfredrik/mrq
  def trace_memory_stop(self):
    """ Stops measuring memory consumption """

    objgraph.show_growth(limit=10)

    trace_type = get_current_config()["trace_memory_type"]
    if trace_type:

      objgraph.show_chain(
        objgraph.find_backref_chain(
          random.choice(objgraph.by_type(trace_type)),
          objgraph.is_proper_module
        ),
        filename='%s/%s-%s.png' % (get_current_config()["trace_memory_output_dir"], trace_type, self.id)
      )

    gc.collect()
    self._memory_stop = self.worker.get_memory()

    diff = self._memory_stop - self._memory_start

    log.debug("Memory diff for job %s : %s" % (self.id, diff))

    # We need to update it later than the results, we need them off memory already.
    self.collection.update({
      "_id": self.id
    }, {"$set": {
      "memory_diff": diff
    }}, w=1)
コード例 #26
0
def show_backref_from_coro():
    import random
    objgraph.show_chain(
        objgraph.find_backref_chain(
            random.choice(objgraph.by_type('coroutine')),
            objgraph.is_proper_module),
        filename='chain-to-leaked.png')
コード例 #27
0
ファイル: objgraphwidget.py プロジェクト: Xelloss-HC/mcedit2
 def showRefs(self):
     objType = str(self.inputWidget.text())
     with self.showTempImage() as fn:
         objgraph.show_refs(objgraph.by_type(objType),
                            filter=self.filterPrimitives,
                            max_depth=self.depthLimitBox.value(),
                            too_many=self.widthLimitBox.value(), filename=fn)
コード例 #28
0
    def trace_memory_stop(self):
        """ Stops measuring memory consumption """

        self.trace_memory_clean_caches()

        objgraph.show_growth(limit=30)

        trace_type = context.get_current_config()["trace_memory_type"]
        if trace_type:

            filename = '%s/%s-%s.png' % (context.get_current_config(
            )["trace_memory_output_dir"], trace_type, self.id)

            chain = objgraph.find_backref_chain(
                random.choice(objgraph.by_type(trace_type)),
                objgraph.is_proper_module)
            objgraph.show_chain(chain, filename=filename)
            del filename
            del chain

        gc.collect()
        self._memory_stop = self.worker.get_memory()["total"]

        diff = self._memory_stop - self._memory_start

        context.log.debug("Memory diff for job %s : %s" % (self.id, diff))

        # We need to update it later than the results, we need them off memory
        # already.
        self.collection.update({"_id": self.id},
                               {"$set": {
                                   "memory_diff": diff
                               }},
                               w=1)
コード例 #29
0
ファイル: gc_utils.py プロジェクト: justinleoye/crawler-group
def show_backrefs_by_type(t):
    o = objgraph.by_type(t)
    if len(o) == 0:
        return False
    obj = o[random.randint(0, len(o) - 1)]
    objgraph.show_backrefs([obj], max_depth=10)
    return True
コード例 #30
0
def get_counts(prefix=''):
    test_types = [
        'Var', 
        'Piecewise',
        'ScenarioTree',
        'ScenarioTreeNode',
        'Scenario',
        '_SetContainer',
        '_ConstraintArray',
        ]
    
    # use objgraph to check if these pyomo objects still exist 
    objects = {} 
    test_counts = {}
    for name in test_types:
        objects[name] = objgraph.by_type( name )
        test_counts[name] = len(objects[name])
    
    if True:
        for name in test_types:
            if test_counts[name]==0: continue
            else: obj = objects[name][0]
            fname = prefix+'-'+'objgraph-{}-'.format(name)
            objgraph.show_refs( [obj], filename=fname+'refs.png') # too_many=50,
            objgraph.show_backrefs([obj], filename=fname+'backref.png')
            objgraph.show_chain(
                objgraph.find_backref_chain( obj, inspect.ismodule),
                filename=fname+'chain.png'
                )
    return test_counts    
コード例 #31
0
ファイル: test_model.py プロジェクト: loicdtx/datacube-core
def test_single_dm_instance(driver_manager, db):
    """
    Our driver manager should only be linked to one PostgresDb instance in memory.

    Subsequent drivers should share the same instance to avoid extra connections (and cache duplication)
    """

    # There's a circular reference in DriverManager structure, so old test instances aren't cleaned up by
    # the reference counter.
    gc.collect()

    # For all PostgresDb instances in memory (there may be others due to other pytests), count
    # how many are connected to our instance of DriverManager
    references = 0
    for pg_instance in objgraph.by_type('PostgresDb'):
        chain = objgraph.find_backref_chain(
            pg_instance,
            max_depth=10,
            predicate=lambda o: isinstance(o, DriverManager))
        # If the referenced DriverManager is ours
        if chain[0] is driver_manager:
            print("Length {}: {}".format(
                len(chain), ' -> '.join(c.__class__.__name__ for c in chain)))
            references += 1

    assert references == 1, "Our DriverManager should only reference one PG instance"
コード例 #32
0
def check_dataframes(
    minimal_memory_usage: float = 10,
    show_views: bool = False,
    show_contents: bool = False,
    show_dtypes: bool = True,
):
    '''
    :param minimal_memory_usage: Minimal memory usage of the variable to be considered. In megabytes.
    '''
    already_printed_base_addresses = []
    for cls in ('DataFrame', 'Series'):
        objs = objgraph.by_type(cls)
        for obj in objs:
            memory_usage = obj.memory_usage(deep=True)
            try:
                memory_usage = memory_usage.sum()
            except:
                pass
            if memory_usage / 1e6 < minimal_memory_usage:
                continue
            object_type = cls
            if obj._is_view:
                memory_usage = 0
                object_type += ' view'
                if not show_views:
                    continue
            # Some view data can be referenced by several views (DataFrame and Series for example). Omit them.
            if id(obj.values.base) in already_printed_base_addresses:
                continue

            backrefs = find_object_in_stack(obj)

            print(
                f'{object_type} found on address {id(obj.values.base)} with memory usage {memory_usage / 1e6} MB. References:'
            )
            print('\n'.join([
                f'\tName = {backref[0]} {backref[1]} in function {backref[3]} in file {backref[2]} '
                for backref in backrefs
            ]))
            if show_contents:
                print(obj)
            if show_dtypes:
                print('\tDtypes =', str(dict(obj.dtypes)))
            # TODO: check if some series aren't sparse to recommend sparse storage
            already_printed_base_addresses.append(id(obj.values.base))

            dtypes = obj.dtypes
            try:
                columns = obj.columns
            except AttributeError:
                # obj is Series
                dtypes = [dtypes]
                columns = ['Series']

            for dtype, column in zip(dtypes, columns):
                if dtype == np.object:
                    print(
                        f'\tWARN: Object dtype found in \'{column}\', this series can take up a lot of memory'
                    )
コード例 #33
0
def test_memory_leak_in_update(sas: StockAnalysisSystem):
    sas = StockAnalysisSystem()
    data_hub = sas.get_data_hub_entry()
    data_center = data_hub.get_data_center()
    data_utility = data_hub.get_data_utility()

    # data_center.update_local_data('Market.SecuritiesInfo', force=False)
    # data_center.update_local_data('Market.NamingHistory')
    # data_center.update_local_data('Market.TradeCalender')

    # stock_list = data_utility.get_stock_list()

    # data_center.update_local_data('Finance.Audit', '600000.SSE', force=True)
    # data_center.update_local_data('Finance.Audit', '600036.SSE', force=True)
    #
    # data_center.update_local_data('Finance.BalanceSheet', '600000.SSE', force=True)
    # data_center.update_local_data('Finance.BalanceSheet', '600036.SSE', force=True)
    #
    # data_center.update_local_data('Finance.IncomeStatement', '600000.SSE', force=True)
    # data_center.update_local_data('Finance.IncomeStatement', '600036.SSE', force=True)
    #
    # data_center.update_local_data('Finance.CashFlowStatement', '600000.SSE', force=True)
    # data_center.update_local_data('Finance.CashFlowStatement', '600036.SSE', force=True)

    data_hub = sas.get_data_hub_entry()
    data_center = data_hub.get_data_center()
    data_utility = data_hub.get_data_utility()
    stock_list = data_utility.get_stock_list()

    counter = 0
    for stock_identity, name in stock_list:
        counter += 1
        data_center.update_local_data('Finance.Audit',
                                      stock_identity,
                                      force=True)
        data_center.update_local_data('Finance.BalanceSheet',
                                      stock_identity,
                                      force=True)
        data_center.update_local_data('Finance.IncomeStatement',
                                      stock_identity,
                                      force=True)
        data_center.update_local_data('Finance.CashFlowStatement',
                                      stock_identity,
                                      force=True)
        data_center.update_local_data('Stockholder.PledgeStatus',
                                      stock_identity,
                                      force=True)
        data_center.update_local_data('Stockholder.PledgeHistory',
                                      stock_identity,
                                      force=True)
        if counter > 100:
            break

    print(gc.collect())
    objs = objgraph.by_type('OBJ')
    if len(objs) > 0:
        objgraph.show_backrefs(objs[0], max_depth=10, filename='obj.dot')

    return True
コード例 #34
0
def run_last():
    run_once("last")
    objgraph.show_chain(
            objgraph.find_backref_chain(
                random.choice(objgraph.by_type('BoundField')),
                objgraph.is_proper_module),
                filename=r'c:\temp\chain.png')
    print( "DONE LAST")
コード例 #35
0
def _check(type_name):
    """Utility function to debug references"""
    import objgraph

    objects = objgraph.by_type(type_name)
    if objects:
        obj = objects[0]
        objgraph.show_backrefs(obj, max_depth=3, filename='graph.png')
コード例 #36
0
def show_memory_refs(name):
    try: obj=objgraph.by_type(name)[0]
    except IndexError:
        print 'no object of type',name  
        return
    objgraph.show_chain(
        objgraph.find_backref_chain(obj, inspect.ismodule),
        filename='chain-{}.png'.format(name))
コード例 #37
0
ファイル: control.py プロジェクト: ryandub/celery
def objgraph(state, num=200, max_depth=10):  # pragma: no cover
    try:
        import objgraph
    except ImportError:
        raise ImportError("Requires the objgraph library")
    with tempfile.NamedTemporaryFile(prefix="cobjg", suffix=".png", delete=False) as fh:
        objects = objgraph.by_type("Request")[:num]
        objgraph.show_backrefs(objects, max_depth=max_depth, highlight=lambda v: v in objects, filename=fh.name)
        return {"filename": fh.name}
コード例 #38
0
ファイル: objgraphwidget.py プロジェクト: Xelloss-HC/mcedit2
 def showBackrefs(self):
     objType = str(self.inputWidget.text())
     with self.showTempImage() as fn:
         objects = objgraph.by_type(objType)
         if len(objects) == 0:
             return
         objgraph.show_backrefs(objects[0],
                                max_depth=self.depthLimitBox.value(),
                                extra_ignore=(id(gc.garbage),id(objects)),
                                too_many=self.widthLimitBox.value(), filename=fn)
コード例 #39
0
ファイル: tests.py プロジェクト: happyhands/objgraph
 def test_new_garbage(self):
     # Regression test for https://github.com/mgedmin/objgraph/pull/22
     gc.disable()
     x = type('MyClass', (), {})()
     res = objgraph.by_type('MyClass')
     self.assertEqual(res, [x])
     # referrers we expect:
     # 1. this stack frame
     # 2. the `res` list
     # referrers we don't want:
     # the ``objects`` list in the now-dead stack frame of objgraph.by_type
     self.assertEqual(len(gc.get_referrers(res[0])), 2)
コード例 #40
0
ファイル: runserver.py プロジェクト: jfsmig/oio-swift
def run_objgraph(types):
    import objgraph
    import os
    import random
    objgraph.show_most_common_types(limit=50, shortnames=False)
    for type_ in types:
        count = objgraph.count(type_)
        print '%s objects: %d' % (type_, count)
        if count:
            objgraph.show_backrefs(
                random.choice(objgraph.by_type(type_)), max_depth=20,
                filename='/tmp/backrefs_%s_%d.dot' % (type_, os.getpid()))
コード例 #41
0
ファイル: memory_usage.py プロジェクト: Adastra-thw/Tortazo
def dump_memory_usage():
    """
    This is a function that prints the memory usage of w3af in real time.
    :author: Andres Riancho ([email protected])
    """
    if not DEBUG_MEMORY:
        return
    else:
        if DEBUG_REFERENCES:
            print 'Object References:'
            print '=================='
            interesting = ['tuple', 'dict', 'list']
            for interesting_klass in interesting:
                interesting_instances = objgraph.by_type(interesting_klass)

                sample = random.sample(interesting_instances, min(
                    SAMPLE_LEN, len(interesting_instances)))

                for s in sample:
                    fmt = 'memory-refs/%s-backref-graph-%s.png'
                    fname = fmt % (interesting_klass, id(s))

                    ignores = [id(interesting_instances), id(s), id(sample)]
                    ignores.extend([id(v) for v in locals().values()])
                    ignores.extend([id(v) for v in globals().values()])
                    ignores.append(id(locals()))
                    ignores.append(id(globals()))
                    try:
                        objgraph.show_backrefs(s, highlight=inspect.isclass,
                                               extra_ignore=ignores, filename=fname,
                                               extra_info=_extra_info)
                    except:
                        pass

            print

        print 'Most common:'
        print '============'
        objgraph.show_most_common_types()

        print

        print 'Memory delta:'
        print '============='
        objgraph.show_growth(limit=25)

        sorted_cmds, shareds, _, _ = get_memory_usage(None, True, True, True)
        cmd = sorted_cmds[0]
        msg = "%8sB Private + %8sB Shared = %8sB" % (human(cmd[1] - shareds[cmd[0]]),
                                                     human(shareds[cmd[0]
                                                                   ]), human(cmd[1])
                                                     )
        print 'Total memory usage:', msg
コード例 #42
0
ファイル: conftest.py プロジェクト: jzuhone/glue
def pytest_unconfigure(config):

    os.environ.pop('GLUE_TESTING')

    # Reset configuration directory to original one
    from glue import config
    config.CFG_DIR = CFG_DIR_ORIG

    # Remove reference to QApplication to prevent segmentation fault on PySide
    try:
        from glue.utils.qt import app
        app.qapp = None
    except ImportError:  # for when we run the tests without the qt directories
        pass

    if OBJGRAPH_INSTALLED and not ON_APPVEYOR:

        # Make sure there are no lingering references to GlueApplication
        obj = objgraph.by_type('GlueApplication')
        if len(obj) > 0:
            objgraph.show_backrefs(objgraph.by_type('GlueApplication'))
            warnings.warn('There are {0} remaining references to GlueApplication'.format(len(obj)))
コード例 #43
0
ファイル: debug.py プロジェクト: amuntner/pappy-proxy
def graph_randobj(line):
    import objgraph
    args = shlex.split(line)
    if len(args) > 1:
        fname = args[1]
    else:
        fname = 'chain.png'
    print 'Getting random %s object...' % args[0]
    obj = random.choice(objgraph.by_type(args[0]))
    print 'Creating chain...'
    chain = objgraph.find_backref_chain(obj, objgraph.is_proper_module)
    print 'Saving chain...'
    objgraph.show_chain(chain, filename=fname)
コード例 #44
0
ファイル: control.py プロジェクト: StackOps/celery
def objgraph(panel, num=200, max_depth=10, ):
    try:
        import objgraph
    except ImportError:
        raise ImportError('Requires the objgraph library')
    with tempfile.NamedTemporaryFile(prefix='cobjg',
                                     suffix='.png', delete=False) as fh:
        objects = objgraph.by_type('Request')[:num]
        objgraph.show_backrefs(
            objects,
            max_depth=max_depth, highlight=lambda v: v in objects,
            filename=fh.name,
        )
        return {'filename': fh.name}
コード例 #45
0
 def get(self):
     count = int(self.request.get('count', '20'))
     max_depth = int(self.request.get('max_depth', '10'))
     type_name = self.request.get('type')
     all_objects = list(objgraph.by_type(type_name))
     # ignore the references from our all_objects container
     ignore = [
         id(all_objects),
         id(sys._getframe(1)),
         id(sys._getframe(1).f_locals),
     ]
     sio = StringIO.StringIO()
     objgraph.show_backrefs(all_objects[:count], max_depth=max_depth, shortnames=False, extra_ignore=ignore, output=sio)
     self.response.headers["Content-Type"] = "text/plain"
     self.response.out.write(sio.getvalue())
コード例 #46
0
ファイル: control.py プロジェクト: 1995rishi/flaskmap
def objgraph(state, num=200, max_depth=10, type='Request'):  # pragma: no cover
    try:
        import objgraph
    except ImportError:
        raise ImportError('Requires the objgraph library')
    print('Dumping graph for type %r' % (type, ))
    with tempfile.NamedTemporaryFile(prefix='cobjg',
                                     suffix='.png', delete=False) as fh:
        objects = objgraph.by_type(type)[:num]
        objgraph.show_backrefs(
            objects,
            max_depth=max_depth, highlight=lambda v: v in objects,
            filename=fh.name,
        )
        return {'filename': fh.name}
コード例 #47
0
ファイル: Levels.py プロジェクト: Norberg/molecule
    def on_key_press(self, symbol, modifiers):
        if symbol == pyglet.window.key.M:
            import objgraph

            objgraph.show_growth()
            m = objgraph.by_type("Molecule")
            print("nr of Molecule:", len(m))
            # for n in m:
            #    objgraph.show_backrefs(n)
        elif symbol == pyglet.window.key.ESCAPE:
            self.window.close()
        elif symbol == pyglet.window.key.S:
            Gui.create_popup(self.window, self.batch, "Skipping level, Cheater!", on_escape=self.window.switch_level)
        elif symbol == pyglet.window.key.R:
            self.window.reset_level()
        elif symbol == pyglet.window.key.D:
            self.window.DEBUG_GRAPHICS = not self.window.DEBUG_GRAPHICS
コード例 #48
0
ファイル: debug.py プロジェクト: mgautierfr/devparrot
 def back_ref():
     import objgraph
     gc.collect()
     #classtypes = ('TextView', 'Document', 'RangeInfo', 'SourceBuffer', 'FileDocSource', 'DocumentView')
     classtypes = ('TopContainer', 'Workspace', 'SplittedContainer', 'NotebookContainer', 'DocumentView')
     #i = 0
     #for o in objgraph.by_type(classtype):
     #    l = objgraph.find_backref_chain(o, objgraph.is_proper_module)
     #    objgraph.show_chain(l, filename="backref%d.png"%i)
     #    i += 1
     #print "test |%s|"% classtype
     #count()
     list_ = []
     for type_ in classtypes:
         list_.extend(objgraph.by_type(type_))
     #print list_
     objgraph.show_backrefs(list_, max_depth=3, too_many=100, filename="backref.png")
コード例 #49
0
ファイル: control.py プロジェクト: jdufresne/celery
def objgraph(state, num=200, max_depth=10, type="Request"):  # pragma: no cover
    """Create graph of uncollected objects (memory-leak debugging).

    Arguments:
        num (int): Max number of objects to graph.
        max_depth (int): Traverse at most n levels deep.
        type (str): Name of object to graph.  Default is ``"Request"``.
    """
    try:
        import objgraph as _objgraph
    except ImportError:
        raise ImportError("Requires the objgraph library")
    logger.info("Dumping graph for type %r", type)
    with tempfile.NamedTemporaryFile(prefix="cobjg", suffix=".png", delete=False) as fh:
        objects = _objgraph.by_type(type)[:num]
        _objgraph.show_backrefs(objects, max_depth=max_depth, highlight=lambda v: v in objects, filename=fh.name)
        return {"filename": fh.name}
コード例 #50
0
ファイル: test_data_viewer.py プロジェクト: sergiopasra/glue
    def teardown_method(self, method):

        if self.viewer is not None:
            self.viewer.close()
        self.application.close()

        # The following is a check to make sure that once the viewer and
        # application have been closed, there are no leftover references to
        # the data viewer. This was introduced because there were previously
        # circular references that meant that viewer instances were not
        # properly garbage collected, which in turn meant they still reacted
        # in some cases to events.
        if OBJGRAPH_INSTALLED:
            self.viewer = None
            self.application = None
            if self.viewer_count > self.viewer_count_start:
                objgraph.show_backrefs(objgraph.by_type(self.viewer_cls.__name__))
                raise ValueError("No net viewers should be created in tests")
コード例 #51
0
ファイル: memory.py プロジェクト: cocrawler/cocrawler
def print_objects(f):
    gc.collect()
    with open(f, 'r') as fd:
        for line in fd:
            line = line.strip()
            try:
                obj = random.choice(objgraph.by_type(line))
            except Exception as e:
                LOGGER.info('exception trying to objgraph a random %s: %s', line, str(e))
                break
            with tempfile.NamedTemporaryFile(dir='/tmp', prefix=line, suffix='.dot', mode='w') as out:
                try:
                    objgraph.show_chain(objgraph.find_backref_chain(obj, objgraph.is_proper_module), output=out)
                    LOGGER.info('object %s file %s', line, out.name)
                except Exception as e:
                    LOGGER.info('exception trying to show_chain a random %s: %s', line, str(e))
    try:
        os.remove(f)
    except Exception as e:
        LOGGER.info('exception %s removing memory_crawler file %s', str(e), f)
コード例 #52
0
ファイル: test_solvers.py プロジェクト: jimmy0415/minpower
def run_one_solver(solver_name):
    prob=simple_problem()
    orig_solver = config.user_config.solver
    config.user_config.duals = False
    config.user_config.solver = solver_name 
    
    prob.solve()
    if mem_tracking: tracker.create_snapshot('prob. solved')
    status=prob.solved
    del prob
    if mem_tracking: tracker.create_snapshot('prob. deleted')
    if solver_name=='glpk' and mem_tracking:
        tracker.stats.print_summary()
        #HtmlStats(tracker=tracker).create_html('profile-simple-problem.html')
        models_left=objgraph.by_type('ConcreteModel')
        print models_left
        objgraph.show_backrefs(models_left,filename='backrefs-simple-problem.png')
    
    config.user_config.duals = True
    config.user_config.solver = orig_solver
    return status
コード例 #53
0
ファイル: tests.py プロジェクト: happyhands/objgraph
 def test_long_type_names(self):
     x = type('MyClass', (), {'__module__': 'mymodule'})()
     self.assertEqual([x], objgraph.by_type('mymodule.MyClass'))
コード例 #54
0
ファイル: objgraph_test.py プロジェクト: UManPychron/pychron
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#===============================================================================

#============= enthought library imports =======================
#============= standard library imports ========================
#============= local library imports  ==========================
import os

path = os.environ['PATH']
os.environ['PATH'] = '{}:/usr/local/bin'.format(path)

import objgraph
import random
import inspect

d = dict(a=1, b='2', c=3.0)
objgraph.show_chain(
                    objgraph.find_backref_chain(
                                                random.choice(objgraph.by_type('dict')),
                                                inspect.ismodule),
                    filename='chain.png')


#============= EOF =============================================
コード例 #55
0
def show_memory_backrefs(name):    
    objgraph.show_backrefs(
        objgraph.by_type(name),
        filename='backrefs-{}.png'.format(name))
コード例 #56
0
def func () :
    js1 = saga.job.Service ('ssh://localhost/bin/sh')


##  js0 = saga.job.Service ('ssh://localhost/bin/sh')
##  t0  = saga.job.Service.create ('ssh://localhost/bin/sh')
##  t0.wait ()
##  js0 = t0.get_result ()
##  print js0.url 

print "--------------------------"
func ()
print "--------------------------"
gc.collect ()

# print "=============================="
# for g in gc.garbage :
#     print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
#     pp (g)
# print "=============================="
# pp (gc.garbage)
# print "=============================="

# checks.show_backrefs (js1._adaptor)
print checks.count('ShellJobService')
checks.show_backrefs(checks.by_type('ShellJobService')[-1])




コード例 #57
0
    def stats(self):
        output = ['Statistics:']

        for trial in range(10):
            if request_counter.count > 0:
                break
            time.sleep(0.5)
        else:
            output.append('\nNot all requests closed properly.')

        # gc_collect isn't perfectly synchronous, because it may
        # break reference cycles that then take time to fully
        # finalize. Call it thrice and hope for the best.
        gc.collect()
        gc.collect()
        unreachable = gc.collect()
        if unreachable:
            if objgraph is not None:
                final = objgraph.by_type('Nondestructible')
                if final:
                    objgraph.show_backrefs(final, filename='finalizers.png')

            trash = {}
            for x in gc.garbage:
                trash[type(x)] = trash.get(type(x), 0) + 1
            if trash:
                output.insert(0, '\n%s unreachable objects:' % unreachable)
                trash = [(v, k) for k, v in trash.items()]
                trash.sort()
                for pair in trash:
                    output.append('    ' + repr(pair))

        # Check declared classes to verify uncollected instances.
        # These don't have to be part of a cycle; they can be
        # any objects that have unanticipated referrers that keep
        # them from being collected.
        allobjs = {}
        for cls, minobj, maxobj, msg in self.classes:
            allobjs[cls] = get_instances(cls)

        for cls, minobj, maxobj, msg in self.classes:
            objs = allobjs[cls]
            lenobj = len(objs)
            if lenobj < minobj or lenobj > maxobj:
                if minobj == maxobj:
                    output.append(
                        '\nExpected %s %r references, got %s.' %
                        (minobj, cls, lenobj))
                else:
                    output.append(
                        '\nExpected %s to %s %r references, got %s.' %
                        (minobj, maxobj, cls, lenobj))

                for obj in objs:
                    if objgraph is not None:
                        ig = [id(objs), id(inspect.currentframe())]
                        fname = 'graph_%s_%s.png' % (cls.__name__, id(obj))
                        objgraph.show_backrefs(
                            obj, extra_ignore=ig, max_depth=4, too_many=20,
                            filename=fname, extra_info=get_context)
                    output.append('\nReferrers for %s (refcount=%s):' %
                                  (repr(obj), sys.getrefcount(obj)))
                    t = ReferrerTree(ignore=[objs], maxdepth=3)
                    tree = t.ascend(obj)
                    output.extend(t.format(tree))

        return '\n'.join(output)