def test_compute_log_subscription(mock_watch_completed): mock_watch_completed.return_value = False schema = create_schema() server = DagsterSubscriptionServer(schema=schema) with instance_for_test() as instance: run = execute_pipeline(example_pipeline, instance=instance) assert run.success assert run.run_id with create_subscription_context(instance) as context: start_subscription( server, context, COMPUTE_LOG_SUBSCRIPTION, { "runId": run.run_id, "stepKey": "example_solid", "ioType": "STDERR", }, ) gc.collect() assert len(objgraph.by_type("SubscriptionObserver")) == 1 assert len(objgraph.by_type("ComputeLogSubscription")) == 1 end_subscription(server, context) gc.collect() assert len(objgraph.by_type("SubscriptionObserver")) == 0 assert len(objgraph.by_type("ComputeLogSubscription")) == 0
def _objgraphs(): import objgraph print(objgraph.show_backrefs(objgraph.by_type("requirement"), filename='/home/anaphora/backrefs.png')) print(objgraph.show_backrefs(objgraph.by_type("Stat"), filename='/home/anaphora/statrefs.png')) print(objgraph.show_backrefs(objgraph.by_type("QueryAPI"), filename='/home/anaphora/dbrefs.png')) print(objgraph.show_backrefs(objgraph.by_type("Noun"), filename='/home/anaphora/nounrefs.png')) print(objgraph.show_refs(objgraph.by_type("Noun"), filename='/home/anaphora/nounrefs2.png'))
def test_graphql_ws_success(instance, empty_app): @pipeline def _test(): pass result = execute_pipeline(_test, instance=instance) run_id = result.run_id # wtf pylint # pylint: disable=not-context-manager with TestClient(empty_app).websocket_connect("/graphql", GraphQLWS.PROTOCOL) as ws: ws.send_json({"type": GraphQLWS.CONNECTION_INIT}) ws.send_json( { "type": GraphQLWS.START, "id": "1", "payload": {"query": EVENT_LOG_SUBSCRIPTION, "variables": {"runId": run_id}}, } ) response = ws.receive_json() assert response["type"] == GraphQLWS.CONNECTION_ACK response = ws.receive_json() assert response["id"] == "1" assert response["type"] == GraphQLWS.DATA gc.collect() assert len(objgraph.by_type("PipelineRunObservableSubscribe")) == 1 # after exiting the context manager and closing the connection gc.collect() assert len(objgraph.by_type("PipelineRunObservableSubscribe")) == 0
def pytest_unconfigure(config): os.environ.pop('GLUE_TESTING') # Reset configuration directory to original one from glue import config config.CFG_DIR = CFG_DIR_ORIG # Remove reference to QApplication to prevent segmentation fault on PySide try: from glue.utils.qt import app app.qapp = None except Exception: # for when we run the tests without the qt directories # Note that we catch any exception, not just ImportError, because # QtPy can raise a PythonQtError. pass if OBJGRAPH_INSTALLED and not ON_APPVEYOR: # Make sure there are no lingering references to GlueApplication obj = objgraph.by_type('GlueApplication') if len(obj) > 0: objgraph.show_backrefs(objgraph.by_type('GlueApplication')) warnings.warn( 'There are {0} remaining references to GlueApplication'.format( len(obj)))
def test_dispose(self, server, load_stubs, spec_dir, grpc_port, protobufs): """ Regression test for client connection part of https://github.com/nameko/nameko-grpc/issues/40 """ stubs = load_stubs("example") clients = {} for _ in range(self.COUNT): client = Client( "//localhost:{}".format(grpc_port), stubs.exampleStub, "none", "high", False, ) proxy = client.start() clients[client] = proxy gc.collect() assert len(objgraph.by_type("ClientConnectionManager")) == self.COUNT for client in clients.keys(): client.stop() gc.collect() assert len(objgraph.by_type("ClientConnectionManager")) == 0
def test_event_log_subscription_chunked(): schema = create_schema() server = DagsterSubscriptionServer(schema=schema) with instance_for_test() as instance, environ( {"DAGIT_EVENT_LOAD_CHUNK_SIZE": "2"}): run = execute_pipeline(example_pipeline, instance=instance) assert run.success assert run.run_id with create_subscription_context(instance) as context: start_subscription(server, context, EVENT_LOG_SUBSCRIPTION, {"runId": run.run_id}) gc.collect() assert len(objgraph.by_type("SubscriptionObserver")) == 1 subs = objgraph.by_type("PipelineRunObservableSubscribe") assert len(subs) == 1 subscription_obj = subs[0] end_subscription(server, context) subscription_obj.stopped.wait(30) subs = None subscription_obj = None gc.collect() assert len(objgraph.by_type("SubscriptionObserver")) == 0 assert len(objgraph.by_type("PipelineRunObservableSubscribe")) == 0
def test_compute_log_subscription(mock_watch_completed): mock_watch_completed.return_value = False with instance_for_test() as instance: run = execute_pipeline(example_pipeline, instance=instance) assert run.success assert run.run_id with create_asgi_client(instance) as client: # pylint: disable=not-context-manager with client.websocket_connect("/graphql", GraphQLWS.PROTOCOL) as ws: start_subscription( ws, COMPUTE_LOG_SUBSCRIPTION, { "runId": run.run_id, "stepKey": "example_solid", "ioType": "STDERR", }, ) gc.collect() assert len(objgraph.by_type("ComputeLogSubscription")) == 1 end_subscription(ws) gc.collect() assert len(objgraph.by_type("ComputeLogSubscription")) == 0
def test_dispose(self, server, load_stubs, spec_dir, grpc_port, protobufs): """ Regression test for server connection part of https://github.com/nameko/nameko-grpc/issues/40 """ stubs = load_stubs("example") clients = {} for _ in range(self.COUNT): client = Client( "//localhost:{}".format(grpc_port), stubs.exampleStub, "none", "high", False, ) proxy = client.start() clients[client] = proxy gc.collect() assert len(objgraph.by_type("ServerConnectionManager")) == self.COUNT for client, proxy in clients.items(): response = proxy.unary_unary(protobufs.ExampleRequest(value="A")) assert response.message == "A" client.stop() # while the server is running there is always exactly one server connnection # that remains in memory -- the previously used one remains in the closure of # the "while is accepting" loop inside GrpcServer.run gc.collect() assert len(objgraph.by_type("ServerConnectionManager")) == 1
def test_generic_subscriptions(): server = DagsterSubscriptionServer(schema=None) with instance_for_test() as instance: with create_subscription_context(instance) as context: start_subscription(server, context) gc.collect() assert len(objgraph.by_type("SubscriptionObserver")) == 1 end_subscription(server, context) gc.collect() assert len(objgraph.by_type("SubscriptionObserver")) == 0
def memory_content(interactive=False): typestats = dict(objgraph.typestats()) while True: types = [] for itype, type_name in enumerate(sorted(typestats.keys())): print '%d %s: %d' % (itype, type_name, typestats[type_name]) types.append(type_name) cls = eval(type_name) if hasattr(cls, '__len__'): for obj in objgraph.by_type(type_name): if len(obj) > 1000: print ' Large object (n=%d) [%s ...]' % (len(obj), str(obj[0])) if not interactive: break print 'Inspect? (0-%d or q):' % itype response = sys.stdin.readline().strip() if response == 'q': break try: itype = int(response) objs = objgraph.by_type(types[itype]) except: print 'Unrecognized input %s' % response continue while True: print 'Item #? (0-%d or q):' % len(objs) response = sys.stdin.readline() if response == 'q': break try: iobj = int(response) obj = objs[iobj] except: print 'Unrecognized input %s' % response continue while True: print 'Expression? (use "obj", e.g., len(obj), or q):' response = sys.stdin.readline() if response == 'q': break try: print eval(response) except: print 'Unrecognized input %s' % response continue
def obj(): _log.debug('Generations: {}'.format(len(objgraph.by_type('Generation')))) _log.debug('Individuals: {}'.format(len(objgraph.by_type('Individual')))) _log.debug('NNAgents: {}'.format(len(objgraph.by_type('NNAgent')))) _log.debug('Sequentials: {}'.format(len(objgraph.by_type('Sequential')))) for i, obj in enumerate(objgraph.by_type('Sequential')): # objgraph.show_chain(objgraph.find_backref_chain(obj, objgraph.is_proper_module), # filename='refs/chain_{}.png'.format(i)) # objgraph.show_backrefs(obj, max_depth=5, too_many=5, filename='refs/backrefs_{}.png'.format(i)) pass
def memory_content(interactive = False): typestats = dict(objgraph.typestats()) while True: types = [] for itype, type_name in enumerate(sorted(typestats.keys())): print '%d %s: %d' % (itype, type_name, typestats[type_name]) types.append(type_name) cls = eval(type_name) if hasattr(cls, '__len__'): for obj in objgraph.by_type(type_name): if len(obj) > 1000: print ' Large object (n=%d) [%s ...]' % (len(obj), str(obj[0])) if not interactive: break print 'Inspect? (0-%d or q):' % itype response = sys.stdin.readline().strip() if response == 'q': break try: itype = int(response) objs = objgraph.by_type(types[itype]) except: print 'Unrecognized input %s' % response continue while True: print 'Item #? (0-%d or q):' % len(objs) response = sys.stdin.readline() if response == 'q': break try: iobj = int(response) obj = objs[iobj] except: print 'Unrecognized input %s' % response continue while True: print 'Expression? (use "obj", e.g., len(obj), or q):' response = sys.stdin.readline() if response == 'q': break try: print eval(response) except: print 'Unrecognized input %s' % response continue
def memory_tracing(key_type: str = "lineno", limit: int = 15): """ Traces memory consumption and prints memory-usage statistics when leaving the context :param key_type: :param limit: :return: """ tracemalloc.start() print("--- Tracing memory... ---") try: # Do computation ... yield None finally: snapshot = tracemalloc.take_snapshot() # snapshot = snapshot.filter_traces(( # tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), # tracemalloc.Filter(False, "<unknown>"), # )) top_stats = snapshot.statistics(key_type) print("--- Memory usage statistics: ---") print("Top %s lines:" % limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) print("#%s: %s:%s: %.1f KiB" % (index, filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(' %s' % line) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) print("%s other: %.1f KiB" % (len(other), size / 1024)) total = sum(stat.size for stat in top_stats) print("\nTotal allocated size: %.1f KiB" % (total / 1024)) # May also be useful: import objgraph print("\nTypes of most common instances:") objgraph.show_most_common_types(limit=limit) print("\nObjects that do not have any referents:") objgraph.get_leaking_objects() print("\nIncrease in peak object counts since last call:") objgraph.show_growth(limit=limit) print("\ntuple objects tracked by the garbage collector:") objgraph.by_type('tuple') print("\ndict objects tracked by the garbage collector:") objgraph.by_type('dict') print("--- End of memory tracing ---")
def checkCycleReference(): try: import objgraph except ImportError: return # for tp in objgraph.by_type("Entity"): # filename = ProjectPath().filepath("output", 'chain_{}.png'.format(time.time())) # objgraph.show_backrefs(tp, max_depth=10, filename=filename) logging.debug(objgraph.by_type("Entity")) filename = ProjectPath().filepath("output", 'chain_{}.png'.format(time.time())) objgraph.show_refs(objgraph.by_type("Entity"), max_depth=10, filename=filename)
def save_graph(self, obj_name, max_depth=5): if self.enable: if type(obj_name) is not str: print('input class name into "obj_name" parameter.') return refs_filepath = os.path.join(self.output_path, obj_name + '.refs..png') backrefs_filepath = os.path.join(self.output_path, obj_name + '.backrefs..png') objgraph.show_backrefs(objgraph.by_type(obj_name), max_depth=max_depth, filename=backrefs_filepath) objgraph.show_refs(objgraph.by_type(obj_name), max_depth=max_depth, filename=refs_filepath)
async def test_memory_objects(server: Server, browser_name: str) -> None: async with async_playwright() as p: browser = await p[browser_name].launch() page = await browser.new_page() await page.goto(server.EMPTY_PAGE) page.on("dialog", lambda dialog: dialog.dismiss()) for _ in range(100): await page.evaluate("""async () => alert()""") await page.route("**/", lambda route, _: route.fulfill(body="OK")) for _ in range(100): response = await page.evaluate("""async () => (await fetch("/")).text()""") assert response == "OK" await browser.close() gc.collect() pw_objects: defaultdict = defaultdict(int) for o in objgraph.by_type("dict"): name = o.get("_type") if not name: continue pw_objects[name] += 1 assert "Dialog" not in pw_objects assert "Request" not in pw_objects assert "Route" not in pw_objects
def test_commits_pickling(): repo_path = os.path.join(settings.get('git_repositories_dir'), 'trfl') repo = RepositoryMining(repo_path) commits = list(repo.traverse_commits())[:10] cnt_before = len(objgraph.by_type("Commit")) print(f'Starting with {cnt_before}') for n, commit in enumerate(commits): pickle.dumps(commit) print(f'#{n+1} {len(objgraph.by_type("Commit"))}') cnt_after = len(objgraph.by_type("Commit")) print(f'Ending with {cnt_after}') assert cnt_before == cnt_after # Track issue on https://github.com/ishepard/pydriller/issues/102
def log_mem_usage(signum, frame, fname=None): global _count _count += 1 gc.collect() if not fname: fname = filename + '_memory_%02d.log' % _count with open(fname, 'wb') as f: f.write('gc.garbage: %d\n\n' % len(gc.garbage)) objgraph.show_most_common_types(limit=50, file=f) f.write('\n\n') buf = StringIO() objgraph.show_growth(limit=50, file=buf) buf = buf.getvalue() f.write(buf) if _count < 2: return for tn, l in enumerate(buf.splitlines()[:10]): l = l.strip() if not l: continue type_ = l.split()[0] objects = objgraph.by_type(type_) objects = random.sample(objects, min(50, len(objects))) objgraph.show_chain( objgraph.find_backref_chain(objects[0], objgraph.is_proper_module), filename=fname[:-4] + '_type_%02d_backref.png' % tn) objgraph.show_backrefs( objects, max_depth=5, extra_info=lambda x: hex(id(x)), filename=fname[:-4] + '_type_%02d_backrefs.png' % tn, )
def print_objects(f): gc.collect() with open(f, 'r') as fd: for line in fd: line = line.strip() try: obj = random.choice(objgraph.by_type(line)) except Exception as e: LOGGER.info('exception trying to objgraph a random %s: %s', line, str(e)) break with tempfile.NamedTemporaryFile(dir='/tmp', prefix=line, suffix='.dot', mode='w') as out: try: objgraph.show_chain(objgraph.find_backref_chain( obj, objgraph.is_proper_module), output=out) LOGGER.info('object %s file %s', line, out.name) except Exception as e: LOGGER.info( 'exception trying to show_chain a random %s: %s', line, str(e)) try: os.remove(f) except Exception as e: LOGGER.info('exception %s removing memory_crawler file %s', str(e), f)
def log_mem_usage(signum, frame, fname=None): global _count _count += 1 gc.collect() if not fname: fname = filename + '_memory_%02d.log' % _count with open(fname, 'wb') as f: f.write('gc.garbage: %d\n\n' % len(gc.garbage)) objgraph.show_most_common_types(limit=50, file=f) f.write('\n\n') buf = StringIO() objgraph.show_growth(limit=50, file=buf) buf = buf.getvalue() f.write(buf) if _count < 2: return for tn, l in enumerate(buf.splitlines()[:10]): l = l.strip() if not l: continue type_ = l.split()[0] objects = objgraph.by_type(type_) objects = random.sample(objects, min(50, len(objects))) objgraph.show_chain( objgraph.find_backref_chain( objects[0], objgraph.is_proper_module), filename=fname[:-4] + '_type_%02d_backref.png' % tn ) objgraph.show_backrefs( objects, max_depth=5, extra_info=lambda x: hex(id(x)), filename=fname[:-4] + '_type_%02d_backrefs.png' % tn, )
async def test_memory_objects() -> None: async with async_playwright() as p: browser = await p.chromium.launch() page = await browser.new_page() await page.goto("https://example.com") page.on("dialog", lambda dialog: dialog.dismiss()) for _ in range(100): await page.evaluate("""async () => alert()""") await page.route("**/", lambda route, _: route.fulfill(body="OK")) for _ in range(100): response = await page.evaluate( """async () => (await fetch("/")).text()""") assert response == "OK" await browser.close() gc.collect() df_dicts = pd.DataFrame() df_dicts["dicts"] = objgraph.by_type("dict") df_dicts["pw_types"] = df_dicts["dicts"].apply(lambda x: x.get("_type")) head = df_dicts["pw_types"].value_counts().head(20) assert "Dialog" not in head.items() assert "Request" not in head.items() assert "Route" not in head.items()
def get_counts(prefix=""): test_types = [ "Var", "Piecewise", "ScenarioTree", "ScenarioTreeNode", "Scenario", "_SetContainer", "_ConstraintArray", ] # use objgraph to check if these pyomo objects still exist objects = {} test_counts = {} for name in test_types: objects[name] = objgraph.by_type(name) test_counts[name] = len(objects[name]) if True: for name in test_types: if test_counts[name] == 0: continue else: obj = objects[name][0] fname = prefix + "-" + "objgraph-{}-".format(name) objgraph.show_refs([obj], filename=fname + "refs.png") # too_many=50, objgraph.show_backrefs([obj], filename=fname + "backref.png") objgraph.show_chain( objgraph.find_backref_chain(obj, inspect.ismodule), filename=fname + "chain.png", ) return test_counts
def teardown_method(self, method): if self.viewer is not None: self.viewer.close() self.application.close() # Matplotlib 3.5 introduced a memory leak when resizing the viewer # in https://github.com/matplotlib/matplotlib/pull/19255 so for now # we skip the affected test for the objgraph testing if method.__name__ == 'test_aspect_resize': return # The following seems to fail on Python 3.10 - to be investigated if sys.version_info[:2] >= (3, 10) and method.__name__ == 'test_session_round_trip': return # The following is a check to make sure that once the viewer and # application have been closed, there are no leftover references to # the data viewer. This was introduced because there were previously # circular references that meant that viewer instances were not # properly garbage collected, which in turn meant they still reacted # in some cases to events. if OBJGRAPH_INSTALLED: self.viewer = None self.application = None if self.viewer_count > self.viewer_count_start: objgraph.show_backrefs(objgraph.by_type(self.viewer_cls.__name__)) raise ValueError("No net viewers should be created in tests")
def run_last(): run_once("last") objgraph.show_chain(objgraph.find_backref_chain( random.choice(objgraph.by_type('BoundField')), objgraph.is_proper_module), filename=r'c:\temp\chain.png') print("DONE LAST")
def trace_memory_stop(self): """ Stops measuring memory consumption """ objgraph.show_growth(limit=10) trace_type = get_current_config()["trace_memory_type"] if trace_type: objgraph.show_chain( objgraph.find_backref_chain( random.choice(objgraph.by_type(trace_type)), objgraph.is_proper_module ), filename='%s/%s-%s.png' % (get_current_config()["trace_memory_output_dir"], trace_type, self.id) ) gc.collect() self._memory_stop = self.worker.get_memory() diff = self._memory_stop - self._memory_start log.debug("Memory diff for job %s : %s" % (self.id, diff)) # We need to update it later than the results, we need them off memory already. self.collection.update({ "_id": self.id }, {"$set": { "memory_diff": diff }}, w=1)
def show_backref_from_coro(): import random objgraph.show_chain( objgraph.find_backref_chain( random.choice(objgraph.by_type('coroutine')), objgraph.is_proper_module), filename='chain-to-leaked.png')
def showRefs(self): objType = str(self.inputWidget.text()) with self.showTempImage() as fn: objgraph.show_refs(objgraph.by_type(objType), filter=self.filterPrimitives, max_depth=self.depthLimitBox.value(), too_many=self.widthLimitBox.value(), filename=fn)
def trace_memory_stop(self): """ Stops measuring memory consumption """ self.trace_memory_clean_caches() objgraph.show_growth(limit=30) trace_type = context.get_current_config()["trace_memory_type"] if trace_type: filename = '%s/%s-%s.png' % (context.get_current_config( )["trace_memory_output_dir"], trace_type, self.id) chain = objgraph.find_backref_chain( random.choice(objgraph.by_type(trace_type)), objgraph.is_proper_module) objgraph.show_chain(chain, filename=filename) del filename del chain gc.collect() self._memory_stop = self.worker.get_memory()["total"] diff = self._memory_stop - self._memory_start context.log.debug("Memory diff for job %s : %s" % (self.id, diff)) # We need to update it later than the results, we need them off memory # already. self.collection.update({"_id": self.id}, {"$set": { "memory_diff": diff }}, w=1)
def show_backrefs_by_type(t): o = objgraph.by_type(t) if len(o) == 0: return False obj = o[random.randint(0, len(o) - 1)] objgraph.show_backrefs([obj], max_depth=10) return True
def get_counts(prefix=''): test_types = [ 'Var', 'Piecewise', 'ScenarioTree', 'ScenarioTreeNode', 'Scenario', '_SetContainer', '_ConstraintArray', ] # use objgraph to check if these pyomo objects still exist objects = {} test_counts = {} for name in test_types: objects[name] = objgraph.by_type( name ) test_counts[name] = len(objects[name]) if True: for name in test_types: if test_counts[name]==0: continue else: obj = objects[name][0] fname = prefix+'-'+'objgraph-{}-'.format(name) objgraph.show_refs( [obj], filename=fname+'refs.png') # too_many=50, objgraph.show_backrefs([obj], filename=fname+'backref.png') objgraph.show_chain( objgraph.find_backref_chain( obj, inspect.ismodule), filename=fname+'chain.png' ) return test_counts
def test_single_dm_instance(driver_manager, db): """ Our driver manager should only be linked to one PostgresDb instance in memory. Subsequent drivers should share the same instance to avoid extra connections (and cache duplication) """ # There's a circular reference in DriverManager structure, so old test instances aren't cleaned up by # the reference counter. gc.collect() # For all PostgresDb instances in memory (there may be others due to other pytests), count # how many are connected to our instance of DriverManager references = 0 for pg_instance in objgraph.by_type('PostgresDb'): chain = objgraph.find_backref_chain( pg_instance, max_depth=10, predicate=lambda o: isinstance(o, DriverManager)) # If the referenced DriverManager is ours if chain[0] is driver_manager: print("Length {}: {}".format( len(chain), ' -> '.join(c.__class__.__name__ for c in chain))) references += 1 assert references == 1, "Our DriverManager should only reference one PG instance"
def check_dataframes( minimal_memory_usage: float = 10, show_views: bool = False, show_contents: bool = False, show_dtypes: bool = True, ): ''' :param minimal_memory_usage: Minimal memory usage of the variable to be considered. In megabytes. ''' already_printed_base_addresses = [] for cls in ('DataFrame', 'Series'): objs = objgraph.by_type(cls) for obj in objs: memory_usage = obj.memory_usage(deep=True) try: memory_usage = memory_usage.sum() except: pass if memory_usage / 1e6 < minimal_memory_usage: continue object_type = cls if obj._is_view: memory_usage = 0 object_type += ' view' if not show_views: continue # Some view data can be referenced by several views (DataFrame and Series for example). Omit them. if id(obj.values.base) in already_printed_base_addresses: continue backrefs = find_object_in_stack(obj) print( f'{object_type} found on address {id(obj.values.base)} with memory usage {memory_usage / 1e6} MB. References:' ) print('\n'.join([ f'\tName = {backref[0]} {backref[1]} in function {backref[3]} in file {backref[2]} ' for backref in backrefs ])) if show_contents: print(obj) if show_dtypes: print('\tDtypes =', str(dict(obj.dtypes))) # TODO: check if some series aren't sparse to recommend sparse storage already_printed_base_addresses.append(id(obj.values.base)) dtypes = obj.dtypes try: columns = obj.columns except AttributeError: # obj is Series dtypes = [dtypes] columns = ['Series'] for dtype, column in zip(dtypes, columns): if dtype == np.object: print( f'\tWARN: Object dtype found in \'{column}\', this series can take up a lot of memory' )
def test_memory_leak_in_update(sas: StockAnalysisSystem): sas = StockAnalysisSystem() data_hub = sas.get_data_hub_entry() data_center = data_hub.get_data_center() data_utility = data_hub.get_data_utility() # data_center.update_local_data('Market.SecuritiesInfo', force=False) # data_center.update_local_data('Market.NamingHistory') # data_center.update_local_data('Market.TradeCalender') # stock_list = data_utility.get_stock_list() # data_center.update_local_data('Finance.Audit', '600000.SSE', force=True) # data_center.update_local_data('Finance.Audit', '600036.SSE', force=True) # # data_center.update_local_data('Finance.BalanceSheet', '600000.SSE', force=True) # data_center.update_local_data('Finance.BalanceSheet', '600036.SSE', force=True) # # data_center.update_local_data('Finance.IncomeStatement', '600000.SSE', force=True) # data_center.update_local_data('Finance.IncomeStatement', '600036.SSE', force=True) # # data_center.update_local_data('Finance.CashFlowStatement', '600000.SSE', force=True) # data_center.update_local_data('Finance.CashFlowStatement', '600036.SSE', force=True) data_hub = sas.get_data_hub_entry() data_center = data_hub.get_data_center() data_utility = data_hub.get_data_utility() stock_list = data_utility.get_stock_list() counter = 0 for stock_identity, name in stock_list: counter += 1 data_center.update_local_data('Finance.Audit', stock_identity, force=True) data_center.update_local_data('Finance.BalanceSheet', stock_identity, force=True) data_center.update_local_data('Finance.IncomeStatement', stock_identity, force=True) data_center.update_local_data('Finance.CashFlowStatement', stock_identity, force=True) data_center.update_local_data('Stockholder.PledgeStatus', stock_identity, force=True) data_center.update_local_data('Stockholder.PledgeHistory', stock_identity, force=True) if counter > 100: break print(gc.collect()) objs = objgraph.by_type('OBJ') if len(objs) > 0: objgraph.show_backrefs(objs[0], max_depth=10, filename='obj.dot') return True
def run_last(): run_once("last") objgraph.show_chain( objgraph.find_backref_chain( random.choice(objgraph.by_type('BoundField')), objgraph.is_proper_module), filename=r'c:\temp\chain.png') print( "DONE LAST")
def _check(type_name): """Utility function to debug references""" import objgraph objects = objgraph.by_type(type_name) if objects: obj = objects[0] objgraph.show_backrefs(obj, max_depth=3, filename='graph.png')
def show_memory_refs(name): try: obj=objgraph.by_type(name)[0] except IndexError: print 'no object of type',name return objgraph.show_chain( objgraph.find_backref_chain(obj, inspect.ismodule), filename='chain-{}.png'.format(name))
def objgraph(state, num=200, max_depth=10): # pragma: no cover try: import objgraph except ImportError: raise ImportError("Requires the objgraph library") with tempfile.NamedTemporaryFile(prefix="cobjg", suffix=".png", delete=False) as fh: objects = objgraph.by_type("Request")[:num] objgraph.show_backrefs(objects, max_depth=max_depth, highlight=lambda v: v in objects, filename=fh.name) return {"filename": fh.name}
def showBackrefs(self): objType = str(self.inputWidget.text()) with self.showTempImage() as fn: objects = objgraph.by_type(objType) if len(objects) == 0: return objgraph.show_backrefs(objects[0], max_depth=self.depthLimitBox.value(), extra_ignore=(id(gc.garbage),id(objects)), too_many=self.widthLimitBox.value(), filename=fn)
def test_new_garbage(self): # Regression test for https://github.com/mgedmin/objgraph/pull/22 gc.disable() x = type('MyClass', (), {})() res = objgraph.by_type('MyClass') self.assertEqual(res, [x]) # referrers we expect: # 1. this stack frame # 2. the `res` list # referrers we don't want: # the ``objects`` list in the now-dead stack frame of objgraph.by_type self.assertEqual(len(gc.get_referrers(res[0])), 2)
def run_objgraph(types): import objgraph import os import random objgraph.show_most_common_types(limit=50, shortnames=False) for type_ in types: count = objgraph.count(type_) print '%s objects: %d' % (type_, count) if count: objgraph.show_backrefs( random.choice(objgraph.by_type(type_)), max_depth=20, filename='/tmp/backrefs_%s_%d.dot' % (type_, os.getpid()))
def dump_memory_usage(): """ This is a function that prints the memory usage of w3af in real time. :author: Andres Riancho ([email protected]) """ if not DEBUG_MEMORY: return else: if DEBUG_REFERENCES: print 'Object References:' print '==================' interesting = ['tuple', 'dict', 'list'] for interesting_klass in interesting: interesting_instances = objgraph.by_type(interesting_klass) sample = random.sample(interesting_instances, min( SAMPLE_LEN, len(interesting_instances))) for s in sample: fmt = 'memory-refs/%s-backref-graph-%s.png' fname = fmt % (interesting_klass, id(s)) ignores = [id(interesting_instances), id(s), id(sample)] ignores.extend([id(v) for v in locals().values()]) ignores.extend([id(v) for v in globals().values()]) ignores.append(id(locals())) ignores.append(id(globals())) try: objgraph.show_backrefs(s, highlight=inspect.isclass, extra_ignore=ignores, filename=fname, extra_info=_extra_info) except: pass print print 'Most common:' print '============' objgraph.show_most_common_types() print print 'Memory delta:' print '=============' objgraph.show_growth(limit=25) sorted_cmds, shareds, _, _ = get_memory_usage(None, True, True, True) cmd = sorted_cmds[0] msg = "%8sB Private + %8sB Shared = %8sB" % (human(cmd[1] - shareds[cmd[0]]), human(shareds[cmd[0] ]), human(cmd[1]) ) print 'Total memory usage:', msg
def pytest_unconfigure(config): os.environ.pop('GLUE_TESTING') # Reset configuration directory to original one from glue import config config.CFG_DIR = CFG_DIR_ORIG # Remove reference to QApplication to prevent segmentation fault on PySide try: from glue.utils.qt import app app.qapp = None except ImportError: # for when we run the tests without the qt directories pass if OBJGRAPH_INSTALLED and not ON_APPVEYOR: # Make sure there are no lingering references to GlueApplication obj = objgraph.by_type('GlueApplication') if len(obj) > 0: objgraph.show_backrefs(objgraph.by_type('GlueApplication')) warnings.warn('There are {0} remaining references to GlueApplication'.format(len(obj)))
def graph_randobj(line): import objgraph args = shlex.split(line) if len(args) > 1: fname = args[1] else: fname = 'chain.png' print 'Getting random %s object...' % args[0] obj = random.choice(objgraph.by_type(args[0])) print 'Creating chain...' chain = objgraph.find_backref_chain(obj, objgraph.is_proper_module) print 'Saving chain...' objgraph.show_chain(chain, filename=fname)
def objgraph(panel, num=200, max_depth=10, ): try: import objgraph except ImportError: raise ImportError('Requires the objgraph library') with tempfile.NamedTemporaryFile(prefix='cobjg', suffix='.png', delete=False) as fh: objects = objgraph.by_type('Request')[:num] objgraph.show_backrefs( objects, max_depth=max_depth, highlight=lambda v: v in objects, filename=fh.name, ) return {'filename': fh.name}
def get(self): count = int(self.request.get('count', '20')) max_depth = int(self.request.get('max_depth', '10')) type_name = self.request.get('type') all_objects = list(objgraph.by_type(type_name)) # ignore the references from our all_objects container ignore = [ id(all_objects), id(sys._getframe(1)), id(sys._getframe(1).f_locals), ] sio = StringIO.StringIO() objgraph.show_backrefs(all_objects[:count], max_depth=max_depth, shortnames=False, extra_ignore=ignore, output=sio) self.response.headers["Content-Type"] = "text/plain" self.response.out.write(sio.getvalue())
def objgraph(state, num=200, max_depth=10, type='Request'): # pragma: no cover try: import objgraph except ImportError: raise ImportError('Requires the objgraph library') print('Dumping graph for type %r' % (type, )) with tempfile.NamedTemporaryFile(prefix='cobjg', suffix='.png', delete=False) as fh: objects = objgraph.by_type(type)[:num] objgraph.show_backrefs( objects, max_depth=max_depth, highlight=lambda v: v in objects, filename=fh.name, ) return {'filename': fh.name}
def on_key_press(self, symbol, modifiers): if symbol == pyglet.window.key.M: import objgraph objgraph.show_growth() m = objgraph.by_type("Molecule") print("nr of Molecule:", len(m)) # for n in m: # objgraph.show_backrefs(n) elif symbol == pyglet.window.key.ESCAPE: self.window.close() elif symbol == pyglet.window.key.S: Gui.create_popup(self.window, self.batch, "Skipping level, Cheater!", on_escape=self.window.switch_level) elif symbol == pyglet.window.key.R: self.window.reset_level() elif symbol == pyglet.window.key.D: self.window.DEBUG_GRAPHICS = not self.window.DEBUG_GRAPHICS
def back_ref(): import objgraph gc.collect() #classtypes = ('TextView', 'Document', 'RangeInfo', 'SourceBuffer', 'FileDocSource', 'DocumentView') classtypes = ('TopContainer', 'Workspace', 'SplittedContainer', 'NotebookContainer', 'DocumentView') #i = 0 #for o in objgraph.by_type(classtype): # l = objgraph.find_backref_chain(o, objgraph.is_proper_module) # objgraph.show_chain(l, filename="backref%d.png"%i) # i += 1 #print "test |%s|"% classtype #count() list_ = [] for type_ in classtypes: list_.extend(objgraph.by_type(type_)) #print list_ objgraph.show_backrefs(list_, max_depth=3, too_many=100, filename="backref.png")
def objgraph(state, num=200, max_depth=10, type="Request"): # pragma: no cover """Create graph of uncollected objects (memory-leak debugging). Arguments: num (int): Max number of objects to graph. max_depth (int): Traverse at most n levels deep. type (str): Name of object to graph. Default is ``"Request"``. """ try: import objgraph as _objgraph except ImportError: raise ImportError("Requires the objgraph library") logger.info("Dumping graph for type %r", type) with tempfile.NamedTemporaryFile(prefix="cobjg", suffix=".png", delete=False) as fh: objects = _objgraph.by_type(type)[:num] _objgraph.show_backrefs(objects, max_depth=max_depth, highlight=lambda v: v in objects, filename=fh.name) return {"filename": fh.name}
def teardown_method(self, method): if self.viewer is not None: self.viewer.close() self.application.close() # The following is a check to make sure that once the viewer and # application have been closed, there are no leftover references to # the data viewer. This was introduced because there were previously # circular references that meant that viewer instances were not # properly garbage collected, which in turn meant they still reacted # in some cases to events. if OBJGRAPH_INSTALLED: self.viewer = None self.application = None if self.viewer_count > self.viewer_count_start: objgraph.show_backrefs(objgraph.by_type(self.viewer_cls.__name__)) raise ValueError("No net viewers should be created in tests")
def print_objects(f): gc.collect() with open(f, 'r') as fd: for line in fd: line = line.strip() try: obj = random.choice(objgraph.by_type(line)) except Exception as e: LOGGER.info('exception trying to objgraph a random %s: %s', line, str(e)) break with tempfile.NamedTemporaryFile(dir='/tmp', prefix=line, suffix='.dot', mode='w') as out: try: objgraph.show_chain(objgraph.find_backref_chain(obj, objgraph.is_proper_module), output=out) LOGGER.info('object %s file %s', line, out.name) except Exception as e: LOGGER.info('exception trying to show_chain a random %s: %s', line, str(e)) try: os.remove(f) except Exception as e: LOGGER.info('exception %s removing memory_crawler file %s', str(e), f)
def run_one_solver(solver_name): prob=simple_problem() orig_solver = config.user_config.solver config.user_config.duals = False config.user_config.solver = solver_name prob.solve() if mem_tracking: tracker.create_snapshot('prob. solved') status=prob.solved del prob if mem_tracking: tracker.create_snapshot('prob. deleted') if solver_name=='glpk' and mem_tracking: tracker.stats.print_summary() #HtmlStats(tracker=tracker).create_html('profile-simple-problem.html') models_left=objgraph.by_type('ConcreteModel') print models_left objgraph.show_backrefs(models_left,filename='backrefs-simple-problem.png') config.user_config.duals = True config.user_config.solver = orig_solver return status
def test_long_type_names(self): x = type('MyClass', (), {'__module__': 'mymodule'})() self.assertEqual([x], objgraph.by_type('mymodule.MyClass'))
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. #=============================================================================== #============= enthought library imports ======================= #============= standard library imports ======================== #============= local library imports ========================== import os path = os.environ['PATH'] os.environ['PATH'] = '{}:/usr/local/bin'.format(path) import objgraph import random import inspect d = dict(a=1, b='2', c=3.0) objgraph.show_chain( objgraph.find_backref_chain( random.choice(objgraph.by_type('dict')), inspect.ismodule), filename='chain.png') #============= EOF =============================================
def show_memory_backrefs(name): objgraph.show_backrefs( objgraph.by_type(name), filename='backrefs-{}.png'.format(name))
def func () : js1 = saga.job.Service ('ssh://localhost/bin/sh') ## js0 = saga.job.Service ('ssh://localhost/bin/sh') ## t0 = saga.job.Service.create ('ssh://localhost/bin/sh') ## t0.wait () ## js0 = t0.get_result () ## print js0.url print "--------------------------" func () print "--------------------------" gc.collect () # print "==============================" # for g in gc.garbage : # print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" # pp (g) # print "==============================" # pp (gc.garbage) # print "==============================" # checks.show_backrefs (js1._adaptor) print checks.count('ShellJobService') checks.show_backrefs(checks.by_type('ShellJobService')[-1])
def stats(self): output = ['Statistics:'] for trial in range(10): if request_counter.count > 0: break time.sleep(0.5) else: output.append('\nNot all requests closed properly.') # gc_collect isn't perfectly synchronous, because it may # break reference cycles that then take time to fully # finalize. Call it thrice and hope for the best. gc.collect() gc.collect() unreachable = gc.collect() if unreachable: if objgraph is not None: final = objgraph.by_type('Nondestructible') if final: objgraph.show_backrefs(final, filename='finalizers.png') trash = {} for x in gc.garbage: trash[type(x)] = trash.get(type(x), 0) + 1 if trash: output.insert(0, '\n%s unreachable objects:' % unreachable) trash = [(v, k) for k, v in trash.items()] trash.sort() for pair in trash: output.append(' ' + repr(pair)) # Check declared classes to verify uncollected instances. # These don't have to be part of a cycle; they can be # any objects that have unanticipated referrers that keep # them from being collected. allobjs = {} for cls, minobj, maxobj, msg in self.classes: allobjs[cls] = get_instances(cls) for cls, minobj, maxobj, msg in self.classes: objs = allobjs[cls] lenobj = len(objs) if lenobj < minobj or lenobj > maxobj: if minobj == maxobj: output.append( '\nExpected %s %r references, got %s.' % (minobj, cls, lenobj)) else: output.append( '\nExpected %s to %s %r references, got %s.' % (minobj, maxobj, cls, lenobj)) for obj in objs: if objgraph is not None: ig = [id(objs), id(inspect.currentframe())] fname = 'graph_%s_%s.png' % (cls.__name__, id(obj)) objgraph.show_backrefs( obj, extra_ignore=ig, max_depth=4, too_many=20, filename=fname, extra_info=get_context) output.append('\nReferrers for %s (refcount=%s):' % (repr(obj), sys.getrefcount(obj))) t = ReferrerTree(ignore=[objs], maxdepth=3) tree = t.ascend(obj) output.extend(t.format(tree)) return '\n'.join(output)