def log_mem_usage(signum, frame, fname=None): global _count _count += 1 gc.collect() if not fname: fname = filename + '_memory_%02d.log' % _count with open(fname, 'wb') as f: f.write('gc.garbage: %d\n\n' % len(gc.garbage)) objgraph.show_most_common_types(limit=50, file=f) f.write('\n\n') buf = StringIO() objgraph.show_growth(limit=50, file=buf) buf = buf.getvalue() f.write(buf) if _count < 2: return for tn, l in enumerate(buf.splitlines()[:10]): l = l.strip() if not l: continue type_ = l.split()[0] objects = objgraph.by_type(type_) objects = random.sample(objects, min(50, len(objects))) objgraph.show_chain( objgraph.find_backref_chain( objects[0], objgraph.is_proper_module), filename=fname[:-4] + '_type_%02d_backref.png' % tn ) objgraph.show_backrefs( objects, max_depth=5, extra_info=lambda x: hex(id(x)), filename=fname[:-4] + '_type_%02d_backrefs.png' % tn, )
def cycle_ref(): x = [] y = [x, [x], dict(x=x)] objgraph.show_refs([y], filename='smaple-graph.png') objgraph.show_backrefs([x], filename='sample-backref-graph.png') objgraph.show_most_common_types()
def sigusr1_handler(signal, frame): logger.info( "######SIGUSR1 Received######") # Unfortunately, the objgraph functions use 'print', so I haven't figured out # how to get them into the logger... objgraph.show_most_common_types() objgraph.show_growth( limit=3) logger.info( "###########################")
def run(self, key, value=None): if value is not None: obj = list(filter(lambda o: str(o) == value, get_objects()))[0] if key == "graph": from objgraph import show_refs if value is None: p = self.console.parent show_refs(self.console if p is None else p, refcounts=True, max_depth=3) else: show_refs(obj, refcounts=True, max_depth=3) elif key == "growth": from objgraph import get_leaking_objects, show_most_common_types show_most_common_types(objects=get_leaking_objects()) elif key == "info": from psutil import Process p = Process(os.getpid()) print_formatted_text(p.memory_info()) elif key == "leaking": from objgraph import get_leaking_objects print_formatted_text(get_leaking_objects()) elif key == "objects": data = [["Object", "#References"]] for o in get_objects(): if isinstance(o, (Console, Module)): data.append([str(o), str(getrefcount(o))]) t = BorderlessTable(data, "Consoles/Modules") print_formatted_text(t.table) elif key == "refs": if value is not None: print_formatted_text(getrefcount(obj), ":") pprint(get_referrers(obj))
def print_top_types(self, limit: int) -> None: """Print top x object in memory Arguments: limit {int} -- max number of results """ show_most_common_types(limit=int(limit))
def log_mem_usage(signum, frame, fname=None): global _count _count += 1 gc.collect() if not fname: fname = filename + '_memory_%02d.log' % _count with open(fname, 'wb') as f: f.write('gc.garbage: %d\n\n' % len(gc.garbage)) objgraph.show_most_common_types(limit=50, file=f) f.write('\n\n') buf = StringIO() objgraph.show_growth(limit=50, file=buf) buf = buf.getvalue() f.write(buf) if _count < 2: return for tn, l in enumerate(buf.splitlines()[:10]): l = l.strip() if not l: continue type_ = l.split()[0] objects = objgraph.by_type(type_) objects = random.sample(objects, min(50, len(objects))) objgraph.show_chain( objgraph.find_backref_chain(objects[0], objgraph.is_proper_module), filename=fname[:-4] + '_type_%02d_backref.png' % tn) objgraph.show_backrefs( objects, max_depth=5, extra_info=lambda x: hex(id(x)), filename=fname[:-4] + '_type_%02d_backrefs.png' % tn, )
def target(export_dir, config, pid): print("===== before =====") # memory_usage(pid) print("=====common types=====") objgraph.show_most_common_types() print("=====common types=====") print("=====growth=====") objgraph.show_growth() print("=====growth=====") print("===== before =====") estimator = MyEstimator(model_dir=config.model_dir, config=config) result_dir = estimator.export_savedmodel(export_dir, serving_input_fn) print("Result dir: ", result_dir) time.sleep(1) print("Show stats:") clean(estimator) print("===== after =====") # memory_usage(pid) print("=====common types=====") objgraph.show_most_common_types() print("=====common types=====") print("=====growth=====") objgraph.show_growth() print("=====growth=====") print("===== after =====")
def _run_worker(self): '''Run a single consumer. Coroutine. ''' _logger.debug('Worker start.') while True: priority, item = yield From(self._item_queue.get()) if item == self.POISON_PILL: _logger.debug('Worker quitting.') return else: _logger.debug(__('Processing item {0}.', item)) self._item_get_semaphore.release() self._token_queue.get_nowait() yield From(self._process_item(item)) self._token_queue.task_done() if os.environ.get('OBJGRAPH_DEBUG'): import gc import objgraph gc.collect() objgraph.show_most_common_types(25) if os.environ.get('FILE_LEAK_DEBUG'): import subprocess output = subprocess.check_output( ['lsof', '-p', str(os.getpid()), '-n']) for line in output.decode('ascii', 'replace').split('\n'): if 'REG' in line and \ (os.getcwd() in line or '/tmp/' in line): print('FILELEAK', line)
def print_summary(f): ''' Log a summary of current memory usage. This is very expensive when there is a lot of memory used. ''' if os.path.isfile(f): print_objects(f) if not config.read('Crawl', 'DebugMemory'): return mem = {} for d in debugs: mem.update(d()) LOGGER.info('Memory summary:') for k in sorted(mem.keys()): v = mem[k] LOGGER.info(' %s len %d bytes %s', k, v['len'], _in_millions(v['bytes'])) LOGGER.info('Top objects:') gc.collect() lines = io.StringIO() objgraph.show_most_common_types(limit=20, file=lines) lines.seek(0) for l in lines.read().splitlines(): LOGGER.info(' %s', l)
async def meminfo(ctx, **_): mem_info = StringIO() objgraph.show_most_common_types(file=mem_info) await util.say(ctx.channel, "```%s```" % mem_info.getvalue()) mem_info = StringIO() objgraph.show_growth(file=mem_info) await util.say(ctx.channel, "```%s```" % mem_info.getvalue())
def test_lots_of_queries(self): import resource import objgraph class LoadTest(Model): k = columns.Integer(primary_key=True) v = columns.Integer() sync_table(LoadTest) gc.collect() objgraph.show_most_common_types() print("Starting...") for i in range(1000000): if i % 25000 == 0: # print memory statistic print("Memory usage: %s" % (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)) LoadTest.create(k=i, v=i) objgraph.show_most_common_types() raise Exception("you shouldn't be here")
def main(): d = {} l = f() d['k'] = l print("list l has {0} objects of type Foo()".format(len(l))) objgraph.show_most_common_types()
def show_leaked_objects(): """ not very useful """ func_that_will_leak() gc.collect() roots = objgraph.get_leaking_objects() print('roots len = %s' % len(roots)) objgraph.show_most_common_types(objects=roots) objgraph.show_refs(roots[:3], refcounts=True, filename='roots.png')
def mem(): collections = cache.get("mtg_coll") cards_name = [] for collection in collections: for card in collection["cards"]: cards_name.append(card["name"]) if request.args.get("measure"): objgraph.show_most_common_types() return json.dumps(cards_name)
def main(): from keras.models import Sequential from keras.layers import Dense from keras.optimizers import Adam from keras.losses import categorical_crossentropy model = Sequential() model.add(Dense(128, input_dim=6 * 54, activation='relu')) model.add(Dense(128, activation='relu')) model.add(Dense(12, activation='softmax')) model.compile(loss=categorical_crossentropy, optimizer=Adam(lr=0.001)) #model.load_weights("./save/mcts_nn_cube.h5") max_random = 5 while True: max_random += 1 def f(model, max_random): for i in range(100): r = 1 + np.random.choice(max_random) print() print("random dist: {}/{}".format(r, max_random), "step:", i) state = State() state.reset_and_randomize(r) mcts = MCTSAgent(model, state, max_depth=100) #print(mcts.initial_node.state) if mcts.is_terminal(): print("Done!") else: mcts.search(steps=10000) prior, _ = mcts.initial_node.state.calculate_priors_and_value( model) prior2 = mcts.initial_node.prior_probabilities probs = mcts.action_probabilities(inv_temp=1) q = mcts.initial_node.mean_action_values model.fit(state.input_array(), probs.reshape((1, 12)), epochs=1, verbose=0) print("Prior:", "[" + "".join(prob_box(p) for p in prior) + "]") print("PrDir:", "[" + "".join(prob_box(p) for p in prior2) + "]") print("Prob: ", "[" + "".join(prob_box(p) for p in probs) + "]") print("Q: ", "[" + "".join(prob_box(max(0, p)) for p in q) + "]") gc.collect() objgraph.show_most_common_types(limit=20) p = Process(target=f, args=(model, max_random)) p.start() p.join() model.save_weights("./save/mcts_nn_cube.h5")
def calculate_rebot_model(args): if args[0] == '--reference': xml = OutputParser().parse(args[1]) else: xml = ResultFromXML(args[0]) p = psutil.Process(os.getpid()) print 'Process memory usage after xml parsing %f M' % (float(p.get_memory_info().rss) / (1024**2)) print 'Most common types' objgraph.show_most_common_types() return xml
def calculate_rebot_model(args): if args[0] == '--reference': xml = OutputParser().parse(args[1]) else: xml = ResultFromXML(args[0]) p = psutil.Process(os.getpid()) print 'Process memory usage after xml parsing %f M' % ( float(p.get_memory_info().rss) / (1024**2)) print 'Most common types' objgraph.show_most_common_types() return xml
def run_objgraph(types): import objgraph import os import random objgraph.show_most_common_types(limit=50, shortnames=False) for type_ in types: count = objgraph.count(type_) print '%s objects: %d' % (type_, count) if count: objgraph.show_backrefs( random.choice(objgraph.by_type(type_)), max_depth=20, filename='/tmp/backrefs_%s_%d.dot' % (type_, os.getpid()))
def dump_memory_usage(): """ This is a function that prints the memory usage of w3af in real time. :author: Andres Riancho ([email protected]) """ if not DEBUG_MEMORY: return else: if DEBUG_REFERENCES: print 'Object References:' print '==================' interesting = ['tuple', 'dict', 'list'] for interesting_klass in interesting: interesting_instances = objgraph.by_type(interesting_klass) sample = random.sample(interesting_instances, min( SAMPLE_LEN, len(interesting_instances))) for s in sample: fmt = 'memory-refs/%s-backref-graph-%s.png' fname = fmt % (interesting_klass, id(s)) ignores = [id(interesting_instances), id(s), id(sample)] ignores.extend([id(v) for v in locals().values()]) ignores.extend([id(v) for v in globals().values()]) ignores.append(id(locals())) ignores.append(id(globals())) try: objgraph.show_backrefs(s, highlight=inspect.isclass, extra_ignore=ignores, filename=fname, extra_info=_extra_info) except: pass print print 'Most common:' print '============' objgraph.show_most_common_types() print print 'Memory delta:' print '=============' objgraph.show_growth(limit=25) sorted_cmds, shareds, _, _ = get_memory_usage(None, True, True, True) cmd = sorted_cmds[0] msg = "%8sB Private + %8sB Shared = %8sB" % (human(cmd[1] - shareds[cmd[0]]), human(shareds[cmd[0] ]), human(cmd[1]) ) print 'Total memory usage:', msg
def on_selection_change(self, *args): #pass if self.selection: #from listscreen import ListScreenItem objgraph.show_growth() print '...' roots = objgraph.get_leaking_objects() objgraph.show_most_common_types(objects=roots) print '...' objgraph.show_refs(roots[:3], refcounts=True, filename='sad.png') #objgraph.show_chain(objgraph.find_backref_chain(self.selection[0].__self__, objgraph.is_proper_module),filename='chain.png') #objgraph.show_backrefs(self.selection[0].__self__, filename='sample-backref-graph.png') print '...'
def memory_tracing(key_type: str = "lineno", limit: int = 15): """ Traces memory consumption and prints memory-usage statistics when leaving the context :param key_type: :param limit: :return: """ tracemalloc.start() print("--- Tracing memory... ---") try: # Do computation ... yield None finally: snapshot = tracemalloc.take_snapshot() # snapshot = snapshot.filter_traces(( # tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), # tracemalloc.Filter(False, "<unknown>"), # )) top_stats = snapshot.statistics(key_type) print("--- Memory usage statistics: ---") print("Top %s lines:" % limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) print("#%s: %s:%s: %.1f KiB" % (index, filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(' %s' % line) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) print("%s other: %.1f KiB" % (len(other), size / 1024)) total = sum(stat.size for stat in top_stats) print("\nTotal allocated size: %.1f KiB" % (total / 1024)) # May also be useful: import objgraph print("\nTypes of most common instances:") objgraph.show_most_common_types(limit=limit) print("\nObjects that do not have any referents:") objgraph.get_leaking_objects() print("\nIncrease in peak object counts since last call:") objgraph.show_growth(limit=limit) print("\ntuple objects tracked by the garbage collector:") objgraph.by_type('tuple') print("\ndict objects tracked by the garbage collector:") objgraph.by_type('dict') print("--- End of memory tracing ---")
def _memprofile(): """ mem profile app """ from wxgtd.main import run run() import gc gc.collect() while gc.collect() > 0: print 'collect' import objgraph objgraph.show_most_common_types(20) import pdb pdb.set_trace()
def objectoriented(): import matplotlib.figure as figure #from matplotlib.figure import Figure from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas for i in range(total_loops): if i % sample_rate == 0: print(i) objgraph.show_most_common_types() fig = figure.Figure() canvas = FigureCanvas(fig) ax = fig.add_subplot(1, 1, 1) test(ax) save_figure_to_ram(fig)
def main(): d = {} l = f() d['k'] = l print("list l has {0} objects of type Foo()".format(len(l))) objgraph.show_most_common_types() objgraph.show_backrefs(random.choice(objgraph.by_type('Foo')), filename="foo_refs.png") objgraph.show_refs(d, filename='sample-graph.png')
def process_event(self, pyg_event_tuple): pyg_event = pyg_event_tuple[0] if pyg_event.type == pygame.QUIT or ( pyg_event.type == pygame.KEYDOWN and pyg_event.key == pygame.K_ESCAPE): # If user clicked close for listener in self.quit_listeners: listener() else: screen_pos = pyg_event_tuple[1] event = Event() event.screen_mouse_position = screen_pos canvas_point_list = self.find_canvases(screen_pos, self.screen) for active_canvas in canvas_point_list: if pyg_event.type == pygame.MOUSEBUTTONDOWN: if pyg_event.button == 1: active_canvas.mouse_pressed(event) if event.consumed: self.mouse_released_canvas = active_canvas elif pyg_event.button == 4: active_canvas.mouse_wheel_scrolled_up(event) elif pyg_event.button == 5: active_canvas.mouse_wheel_scrolled_down(event) elif pyg_event.type == pygame.MOUSEBUTTONUP: if pyg_event.button == 1: active_canvas.mouse_released(event) if event.consumed and active_canvas is self.mouse_released_canvas: self.mouse_released_canvas = None elif pyg_event.type == pygame.KEYDOWN: if pyg_event.key == pygame.K_LEFT: active_canvas.left_key_pressed(event) elif pyg_event.key == pygame.K_RIGHT: active_canvas.right_key_pressed(event) elif pyg_event.key == pygame.K_UP: active_canvas.up_key_pressed(event) elif pyg_event.key == pygame.K_DOWN: active_canvas.down_key_pressed(event) elif pyg_event.key == pygame.K_0: gc.collect() print("\nprinting most common types..") objgraph.show_most_common_types(limit=10) # x = [] # y = [x, [x], dict(x=x)] # gc.collect() # objgraph.show_refs([y]) if event.consumed: break if pyg_event.type == pygame.MOUSEBUTTONUP and \ self.mouse_released_canvas is not None and \ pyg_event.button == 1: self.mouse_released_canvas.mouse_canceled(event)
def _memprofile(): """ mem profile app """ from tbviewer import main main.run() import gc gc.collect() while gc.collect() > 0: print('collect') import objgraph objgraph.show_most_common_types(20) import pdb pdb.set_trace()
def _memprofile(): ''' mem profile app ''' from photomagick.main import main main() import gc gc.collect() while gc.collect() > 0: print 'collect' import objgraph objgraph.show_most_common_types(20) import pdb pdb.set_trace()
def record_log(self, idle=1): """ 5 分钟打印一次 日志 记录抓取日志,用于调整各个线程参数设置 """ while True: print( 'time_now:%s; sending_queue:%d; sended_queue:%d; response_queue:%d; store_queue:%d' % (str(datetime.today()), self.sending_queue.qsize(), self.sended_queue.qsize(), self.response_queue.qsize(), self.store_queue.qsize())) objgraph.show_most_common_types() time.sleep(300) if self.is_finish(): break
def memory_profile(self): global LAST_HEAP print "{:=^78}".format(' profile begin') import gc gc.collect() print "{:-^78}".format(' growth') print objgraph.show_growth(limit=5) print "{:-^78}".format(' common types') print objgraph.show_most_common_types(limit=5) if LAST_HEAP: leftover = HEAP.heap() - LAST_HEAP print leftover.byrcs[0].byid LAST_HEAP = HEAP.heap() print "{:=^78}".format(' profile end')
def pyplot(close=False, total_close=False): import matplotlib.pyplot as plt for i in range(total_loops): if i % sample_rate == 0: print(i) objgraph.show_most_common_types() fig = plt.figure() ax = fig.add_subplot(1, 1, 1) test(ax) save_figure_to_ram(fig) if close: fig.clf() if total_close: plt.close()
def main(): from keras.models import Sequential from keras.layers import Dense from keras.optimizers import Adam from keras.losses import categorical_crossentropy model = Sequential() model.add(Dense(128, input_dim=6*54, activation='relu')) model.add(Dense(128, activation='relu')) model.add(Dense(12, activation='softmax')) model.compile(loss=categorical_crossentropy, optimizer=Adam(lr=0.001)) #model.load_weights("./save/mcts_nn_cube.h5") max_random = 5 while True: max_random += 1 def f(model, max_random): for i in range(100): r = 1 + np.random.choice(max_random) print() print("random dist: {}/{}".format(r, max_random), "step:", i) state = State() state.reset_and_randomize(r) mcts = MCTSAgent(model, state, max_depth=100) #print(mcts.initial_node.state) if mcts.is_terminal(): print("Done!") else: mcts.search(steps = 10000) prior, _ = mcts.initial_node.state.calculate_priors_and_value(model) prior2 = mcts.initial_node.prior_probabilities probs = mcts.action_probabilities(inv_temp = 1) q = mcts.initial_node.mean_action_values model.fit(state.input_array(), probs.reshape((1,12)), epochs=1, verbose=0) print("Prior:", "[" + "".join(prob_box(p) for p in prior) + "]") print("PrDir:", "[" + "".join(prob_box(p) for p in prior2) + "]") print("Prob: ", "[" + "".join(prob_box(p) for p in probs) + "]") print("Q: ", "[" + "".join(prob_box(max(0,p)) for p in q) + "]") gc.collect() objgraph.show_most_common_types(limit=20) p = Process(target=f, args=(model, max_random)) p.start() p.join() model.save_weights("./save/mcts_nn_cube.h5")
def get(self, name=None): import gc import objgraph # 强制进行垃圾回收 gc.collect() # 打印出对象数目最多的 50 个类型信息 objgraph.show_most_common_types(limit=5) # if self.request.uri in FUCK_THE_SPIDER: for i in FUCK_THE_SPIDER: if self.request.uri.startswith(i): self.redirect('http://zt.bdinfo.net/speedtest/wo3G.rar') print('!!!!f**k: ' + self.request.uri) return if name.endswith('.md'): name = removeSuffix(name) self.redirect('/' + name, permanent=True) return if name.endswith('.html'): name = removeSuffix(name) self.redirect('/' + name, permanent=True) return if name is None or name == '': mds = search(MD_PATH, '*.md', NOT_IN) name = removeSuffix(mds[0][0]) del mds url_name = urllib.parse.quote(name) self.redirect('/' + url_name) else: content = getContent(name) content = md.convert(content) # print(md.toc) modify_time = getModifyTime(name) pre, old = preAndOld(name) self.render('./blog.html', title=name, content=content, time=time, modify_time=modify_time, pre=pre, old=old, author=AUTHOR, author_link=AUTHOR_LINK, toc=md.toc, urllib=urllib)
def print_stats(process, dispatches, uptime): memory_usage = process.memory_full_info().uss memory_usage = memory_usage / 1024**2 cpu_usage = process.cpu_percent() / psutil.cpu_count() print(f"CPU usage: {cpu_usage:.2f}%") print(f"Memory Usage: {memory_usage:.2f} MiB") for key, value in dispatches.items(): print(f"Worker ID {key} Dispatches: {value}") total = sum(dispatches.values()) delta = datetime.datetime.utcnow() - uptime seconds = delta.total_seconds() or 1.0 events_per_second = total / seconds print(f"Total: {total} ({events_per_second:.2f}/s)") print(f"Common Types:") objgraph.show_most_common_types()
def temp_save(q, kb_res): global spider_count mutex.acquire() temp_data[q] = kb_res if spider_count % 100 == 0: print('save...') with open('datasets/all_cqa_question_to_kb_data.json', 'w') as f: json.dump(temp_data, f) gc.collect() objgraph.show_most_common_types(limit=50) spider_count = spider_count + 1 mutex.release()
def parse_category(self, category): self.lock.acquire() try: page, start = self.read_page(category.id) start = int(start) if start else 0 page = int(page) if page else 1 print 'starting category %s from %s page %s item' % (category.id, page, start) baselink = category.link while True: if page>1: link = baselink.replace('.html', '/%s.html' % page) else: link = baselink items = self.getItems(link) if not items: print 'no items on the page %s at %s' % (page, link) break items = items[start:] threads = [] lock = threading.Semaphore(self.concurrent_products) for item in items: th = threading.Thread(target=self.parse_items_thread, args=(lock, category, [item],)) th.start() threads.append(th) for thread in threads: thread.join() del thread start += 1 self.write_page(category.id, page, start) del threads del lock page += 1 start = 0 self.write_page(category.id, page, start) gc.collect() objgraph.show_most_common_types(limit=30) except: print 'Error while parsing category' traceback.print_exc() pass self.lock.release() connection.close() print 'Category %s finished' % category.id
def get_info(): gc.collect() snapshot2 = tracemalloc.take_snapshot() with open("flaskMemoy.log", 'a+') as f: f.write("====================================") f.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) f.write("\n") top_stats = snapshot2.compare_to(b_snapshot, 'lineno') for stat in top_stats[:10]: f.write(str(stat)) f.write("\n") f.write("====================================") f.write("\n") objgraph.show_most_common_types(limit=5, file=f) f.write("====================================") f.write("\n") return jsonify(text="ok")
def test_lots_of_queries(): sync_table(LoadTest) import objgraph gc.collect() objgraph.show_most_common_types() print("Starting...") for i in range(1000000): if i % 25000 == 0: # print memory statistic print("Memory usage: %s" % (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)) LoadTest.create(k=i, v=i) objgraph.show_most_common_types() raise Exception("you shouldn't be here")
def test_lots_of_queries(): sync_table(LoadTest) import objgraph gc.collect() objgraph.show_most_common_types() print "Starting..." for i in range(1000000): if i % 25000 == 0: # print memory statistic print "Memory usage: %s" % (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) LoadTest.create(k=i, v=i) objgraph.show_most_common_types() raise Exception("you shouldn't be here")
def debugrequest(self, event): try: self.log("Event: ", event.__dict__, lvl=critical) if event.action == "storejson": self.log("Storing received object to /tmp", lvl=critical) fp = open( '/tmp/hfosdebugger_' + str(event.user.useruuid) + "_" + str(uuid4()), "w") json.dump(event.data, fp, indent=True) fp.close() if event.action == "memdebug": self.log("Memory hogs:", lvl=critical) objgraph.show_most_common_types(limit=20) if event.action == "growth": self.log("Memory growth since last call:", lvl=critical) objgraph.show_growth() if event.action == "graph": objgraph.show_backrefs([self.root], max_depth=42, filename='backref-graph.png') self.log("Backref graph written.", lvl=critical) if event.action == "exception": class TestException(BaseException): pass raise TestException if event.action == "heap": self.log("Heap log:", self.heapy.heap(), lvl=critical) if event.action == "buildfrontend": self.log("Sending frontend build command") self.fireEvent(frontendbuildrequest(force=True), "setup") if event.action == "logtail": self.fireEvent( logtailrequest(event.user, None, None, event.client), "logger") except Exception as e: self.log("Exception during debug handling:", e, type(e), lvl=critical)
def print_memory(i): global object_counts print("\n\n--------------------- MEMORY -------------------------\n") print("TOTAL OBJECTS\n") o = len(gc.get_objects()) print(o) object_counts.append(o) del o print("\n") print("GROWTH\n") objgraph.show_growth() print("\n") print("COMMON TYPES\n") objgraph.show_most_common_types() print("\n") print("LEAKING OBJECTS\n") roots = objgraph.get_leaking_objects() print("\n") log.info("ROOTS pre-collect : {}\n".format(len(roots))) print("COMMON TYPES IN ROOTS\n") objgraph.show_most_common_types(objects=roots) print("\n") objgraph.show_refs(roots[:3], refcounts=True, filename=TEST_PATH + '/roots_' + str(i) + '.png') print("\n") log.info("Garbage pre collect: " + str(len(gc.garbage))) gc.collect() log.info("Garbage post collect: " + str(len(gc.garbage))) print("\n") roots = objgraph.get_leaking_objects() log.info("ROOTS post-collect : {}".format(len(roots))) print("\n\n---------------------------------------------------\n")
def __dbg_mem(strn): if __DBGMEM: import gc, objgraph print print '#' * 80 print '#' * 80 print '##', strn print 'Collect', gc.collect() print 'Collect', gc.collect() roots = objgraph.get_leaking_objects() if roots: print len(roots) objgraph.show_most_common_types(objects=roots) objgraph.show_refs(roots[:3], refcounts=True, filename='tmp/%s.png' % strn.lower()) else: print 'Nothing' print '#' * 80 print '#' * 80 print
def sortpro(eachapp,eachmem): all_pro = psutil.get_process_list() for each_pro in all_pro: r_each_pro = str(each_pro) pro_n = re.search(r'\'.+\..+\'',r_each_pro) if pro_n: p_ln = pro_n.group() p_ln = p_ln[1:-1] if p_ln.lower() == eachapp: memdect(each_pro,eachmem) else: pass else: pass gc.collect() f1 = open("D:\\pylog.txt",'a') sys.stdout = f1 objgraph.show_most_common_types(limit=30) print "=================================" return
def test_reid_writing(): gallery_dataset = dataset_builder.build(gallery_cfg) query_dataset = dataset_builder.build(gallery_cfg) gallery_dataloader = torch.utils.data.DataLoader(gallery_dataset, batch_size=10, num_workers=0) model_cfg = model_cfg_builder.build(cfg) model = model_builder.build(model_cfg[0]) model = DataParallel(model) output_file = tempfile.mktemp() objgraph.show_growth() write_to_h5(gallery_dataloader, model, output_file) objgraph.show_most_common_types() objgraph.show_growth() time.sleep(1) query_dataloader = torch.utils.data.DataLoader(query_dataset, batch_size=10, num_workers=0) output_file = tempfile.mktemp() write_to_h5(query_dataloader, model, output_file)
def debugrequest(self, event): try: self.log("Event: ", event.__dict__, lvl=critical) if event.action == "storejson": self.log("Storing received object to /tmp", lvl=critical) fp = open('/tmp/hfosdebugger_' + str( event.user.useruuid) + "_" + str(uuid4()), "w") json.dump(event.data, fp, indent=True) fp.close() if event.action == "memdebug": self.log("Memory hogs:", lvl=critical) objgraph.show_most_common_types(limit=20) if event.action == "growth": self.log("Memory growth since last call:", lvl=critical) objgraph.show_growth() if event.action == "graph": objgraph.show_backrefs([self.root], max_depth=42, filename='backref-graph.png') self.log("Backref graph written.", lvl=critical) if event.action == "exception": class TestException(BaseException): pass raise TestException if event.action == "heap": self.log("Heap log:", self.heapy.heap(), lvl=critical) if event.action == "buildfrontend": self.log("Sending frontend build command") self.fireEvent(frontendbuildrequest(force=True), "setup") if event.action == "logtail": self.fireEvent(logtailrequest(event.user, None, None, event.client), "logger") except Exception as e: self.log("Exception during debug handling:", e, type(e), lvl=critical)
def __call__(self, environ, start_response): with self.lock: profiler = cProfile.Profile() def run_app(*a, **kw): self.response = self.app(environ, start_response) profiler.runcall(run_app, environ, start_response) profiler.snapshot_stats() stats = pstats.Stats(profiler) stats.sort_stats('calls') #cumulative # Redirect output out = StringIO() stats.stream = out stats.print_stats() resp = ''.join(self.response) # Lets at least only put this on html-like responses. if resp.strip().startswith('<'): ## The profiling info is just appended to the response. ## Browsers don't mind this. resp += ('<pre style="text-align:left; ' 'border-top: 4px dashed red; padding: 1em;">') resp += cgi.escape(out.getvalue(), True) ct = objgraph.show_most_common_types() print ct resp += ct if ct else '---' output = StringIO() pprint.pprint(environ, output, depth=3) resp += cgi.escape(output.getvalue(), True) resp += '</pre>' return resp
def checkmem(): objgraph.show_most_common_types(limit=5) except ImportError:
def modelCheck(self): """ This method modifies the current state """ backtrack = False path_end_cached_state = False # c = 0 if MEMORY_PROFILING: objgraph.show_growth() while len(self.state_stack) > 0: log.warning("------> starting model checker transition (stack len: %d) <-------" % len(self.state_stack)) start_state = self.state_stack.pop() self.state = start_state if not config.get("runtime.replay"): start_state.restoreState(self.strategy) # self._debugDumpStateStack() log.debug("Current:\n%s" % start_state) if MEMORY_PROFILING and self.good_transitions_count > 500: uo = gc.collect() log.debug("Garbage collector found %d unreachable objects" % uo) objgraph.show_growth() objgraph.show_most_common_types() # Use the code below to get a graph of the references to a # random object of a particular type. Install xdot for best results. # objgraph.show_chain(objgraph.find_backref_chain( # random.choice(objgraph.by_type('EateController')), # inspect.ismodule)) utils.crash("memory testing") # c += 1 # pprint(state_list, file("state_list%d.txt" % c, "w"), width=100) # print c # replay the list of actions if necessary if backtrack: log.info("Backtracking, previous path early termination: %s" % str(path_end_cached_state)) path_end_cached_state = False pop = None if self.useDpor: stats.pushProfile("dpor") start_state.model = self.initial_state.model start_state.invariant_dispatcher = self.initial_state.invariant_dispatcher pop, start_state = self.dpor.startBacktracking(start_state) stats.popProfile() self.state = start_state if config.get("runtime.replay"): replayed_state = self.replayActions(start_state) # Sanity checks if self.state_debug and replayed_state != start_state: pprint(replayed_state, file("state_now.txt", "w"), width=100) pprint(start_state, file("state_before.txt", "w"), width=100) utils.crash("State after replay is different than original (debug enabled)") start_state = replayed_state self.state = start_state if pop: path_end_cached_state = True if self.packetTranslator is not None: self.packetTranslator.stopSniffing() start_state.testPoint("path_end", cached_state=path_end_cached_state) start_state.checkNewViolations() # clean the new_violation state continue next_action = self.chooseAction(start_state) log.debug("Next action: " + str(next_action)) # when dpor decided that the available enabled actions do not need to be executed if self.useDpor and next_action is None: backtrack = True path_end_cached_state = True if self.packetTranslator is not None: self.packetTranslator.stopSniffing() start_state.testPoint("path_end", cached_state=path_end_cached_state) start_state.checkNewViolations() # clean the new_violation state continue if self.state_debug and backtrack: for a in self.getEnabledActions(start_state): assert a in self.getEnabledActions(start_state) or a.node_name == "model_checker" if self.graph != None: self.graph.startTransition(start_state) # Performing the transition would make us to go over the cutoff limit, backtrack if config.get("model.cutoff") > 0 and start_state.path_length+1 > config.get("model.cutoff"): log.warning("Path cutoff limit reached") path_end_cached_state = True backtrack = True if self.packetTranslator is not None: self.packetTranslator.stopSniffing() start_state.testPoint("path_end", cached_state=path_end_cached_state) start_state.checkNewViolations() # clean the new_violation state continue # make a copy of the model before modifying it new_state = None if not config.get("runtime.replay") or self.state_debug: new_state = start_state.copy() new_state.restoreState() else: new_state = start_state start_state = new_state.minimal_copy() if self.state_debug: new_state.state_replay_list += [start_state] self.state = new_state # add the current state back to the stack, if there is still something to explore if start_state.hasAvailableActions(): if not config.get("runtime.replay") or self.state_debug: start_state.storeState(self.strategy) self.state_stack.append(start_state) if next_action.target == "port_stats_special": self.portStatsSpecial(new_state, next_action.args[0]) self.checkManageFaults(next_action, new_state) new_state.testPoint("transition_start") ### Execute the transition ### new_state.executeAction(next_action, start_state.path_length+1) if self.graph != None: self.graph.endTransition(new_state, str(next_action)) new_state.testPoint("transition_end") if new_state.checkNewViolations(): # Check if we need to go on or backtrack log.info("Invariant violations reported, path exploration stopped") path_end_cached_state = True backtrack = True if self.packetTranslator is not None: self.packetTranslator.stopSniffing() new_state.testPoint("path_end", cached_state=path_end_cached_state) new_state.checkNewViolations() # clean the new_violation state continue if self.max_path_length < new_state.path_length: self.max_path_length = new_state.path_length self.good_transitions_count += 1 # It we reached a new state, hash it and put on top of the stack new_state.setAvailableActions(self.getEnabledActions(new_state)) state_hash = hash(new_state) if not state_hash in self.unique_states: self.unique_states.add(state_hash) if new_state.hasAvailableActions(): if not config.get("runtime.replay") or self.state_debug: new_state.storeState(self.strategy) self.state_stack.append(new_state) backtrack = False log.debug("New state with ID: %d" % new_state.state_id) else: log.info("No more actions available, end state reached.") backtrack = True if self.packetTranslator is not None: self.packetTranslator.stopSniffing() new_state.testPoint("path_end", cached_state=path_end_cached_state) new_state.checkNewViolations() # clean the new_violation state else: log.info("Reached a known state, stopping path exploration") path_end_cached_state = True self.old_states_count += 1 backtrack = True if self.packetTranslator is not None: self.packetTranslator.stopSniffing() new_state.testPoint("path_end", cached_state=path_end_cached_state) new_state.checkNewViolations() # clean the new_violation state new_state.testPoint("path_end", cached_state=path_end_cached_state) new_state.checkNewViolations() # clean the new_violation state if self.packetTranslator is not None: self.packetTranslator.destroyTopo()
def get_info_from_stream(section_name): tries = 0 while True: if PRINT_MEMORY_INFO: if section_name == "iem": gc.collect() logger.debug("NEW GRAPH - " + time.strftime("%c")) graphOutput = StringIO() objgraph.show_most_common_types(file=graphOutput) logger.debug(graphOutput.getvalue()) try: stream_url = stream_confs.get(section_name, 'stream_url') stream_obj = Stream.objects.filter(url=stream_url) if not stream_obj: stream_obj = Stream(url=stream_url, name=stream_confs.get( section_name, 'stream_name')) stream_obj.save() else: stream_obj = stream_obj[0] logger.debug("Got stream object for: " + section_name) stream_data = get_stream(stream_url) logger.debug("Done getting stream for: " + section_name) set_up_bracket(section_name, stream_obj) if not stream_data: if stream_obj.up and tries <= 2: logger.debug("Failed getting up stream, stream_url: " + stream_url + " tries: " + str(tries)) tries += 1 time.sleep(5) continue logger.info( "There doesn't seem to be any stream available for: " + stream_url) tries = 0 stream_obj.up = False stream_obj.save() time.sleep(60) continue else: tries = 0 headers = {'Client-ID': 'jy4zwuphqfdvh2nfygxkzb66z23wjz', 'Accept': 'application/vnd.twitchtv.v3+json'} res = requests.get( "https://api.twitch.tv/kraken/channels/" + stream_url, headers=headers) json_res = res.json() if json_res["game"] != "StarCraft II": logger.info( "Current game is not SCII for stream: " + stream_url) stream_obj.up = False stream_obj.save() time.sleep(60) continue if any(x in json_res["status"].lower() for x in ["rerun", "rebroadcast"]): logger.info("Showing rerun in stream: " + stream_url) stream_obj.up = False stream_obj.save() time.sleep(60) continue # At this point we know we have a stream that is up and we need to # parse it stream_obj.up = True stream_obj.save() # Get game objects game_objects = Game.objects.filter(stream=stream_obj) if not game_objects: game = Game(stream=stream_obj) game.save() else: game = game_objects[0] im = get_screenshot(stream_data, section_name) display_type = stream_confs.get(section_name, "display_types") my_data = get_data_from_image(im, display_type) for key in my_data.keys(): logger.debug(str(key) + ": " + str(my_data[key])) if 'l_name' not in my_data: logger.info("No correct conf for: " + stream_url) stream_obj.up = False stream_obj.save() time.sleep(60) continue if not game_live(my_data): if game.game_on: game.game_off_time = timezone.now() game.game_on = False game.save() time.sleep(10) logger.info("No live game for: " + stream_url) continue else: game.game_on = True if get_score(my_data) != 100: save_main_image(display_type, im) players = get_players(stream_obj) p_l = players[0] p_r = players[1] categories = ['name', 'score', 'supply', 'minerals', 'gas', 'workers', 'army'] for category in categories: if 'l_' + category in my_data: setattr(p_l, category, my_data['l_' + category]) setattr(p_r, category, my_data['r_' + category]) p_l.save() p_r.save() # Game stuff if 'time' in my_data: game.current_time = my_data['time'] if 'map' in my_data: game.on_map = my_data['map'] game.save() logger.info("Done with loop for: " + stream_url) time.sleep(2) except Exception as _: logger.error("Something went wrong for stream: " + stream_url, exc_info=True)
def show_types(): if objgraph is None: return "objgraph not installed" objgraph.show_most_common_types(limit=20) return "Printed to console"
def printMemoryCount(): result = {} for o in gc.get_objects(): t = type(o) #print(t) count = result.get(t, 0) result[t] = count + 1 for k,v in result.items(): print(k,v) return result if __name__ == "__main__": objgraph.show_most_common_types() _entries = [Dummy()] * 9999999 print('test') objgraph.show_most_common_types() #memory_tracker = tracker.SummaryTracker() objgraph.show_growth(limit=10) #table = testCase() #memory_tracker.print_diff() printMemoryCount() ##objgraph.show_growth(limit=10) #import inspect, random
log = logging.getLogger(__name__) class Worker(multiprocessing.Process): queue = queue def run(self): #log.warn("run") print thread.get_ident() count = queue.get() print 'In %s, count: %s' % (self.name, count) self.queue.put(count+1) time.sleep(random.randint(2, 3)) count = queue.get() self.queue.put(count-1) print 'In %s, count: %s' % (self.name, count-1) return if __name__ == '__main__': jobs = [] for i in range(5): p = Worker() jobs.append(p) count = queue.get() print '@@', i, count queue.put(count) p.start() for j in jobs: j.join() objgraph.show_most_common_types(limit=20)