def file_test(rows=500000, cols=50): "File test" print("Creating file with {} rows and {} columns".format(rows, cols)) file = create_file(rows, cols) print("Size of the file: {:.2f} MiB".format(getsize(file) / (1024 * 1024))) print("Reading file") sum1 = summarize(get_objects()) las = read(file) sum2 = summarize(get_objects()) diff = get_diff(sum1, sum2) print_(diff) for curve in las.curves: print("Name: {}, Min: {:.2f}, Mean: {:.2f}, Max: {:.2f}" .format(curve.mnemonic, nanmin(curve.data), nanmean(curve.data), nanmax(curve.data))) del las las = read(file) del las las = read(file) del las las = read(file) del las print("Happy end")
def __exit__(self, exc_type, exc_val, exc_tb): if self.debug: try: gc.collect() end_memory = self.process.memory_info().rss net_memory = end_memory-self.start_memory if net_memory > 100 * 1000 * 1000: Log.warning( "MEMORY WARNING (additional {{net_memory|comma}}bytes): "+self.description, default_params=self.params, net_memory=net_memory ) from pympler import summary from pympler import muppy sum1 = sorted(summary.summarize(muppy.get_objects()), key=lambda r: -r[2])[:30] Log.warning("{{data}}", data=sum1) elif end_memory > 1000*1000*1000: Log.warning( "MEMORY WARNING (over {{end_memory|comma}}bytes): "+self.description, default_params=self.params, end_memory=end_memory ) from pympler import summary from pympler import muppy sum1 = sorted(summary.summarize(muppy.get_objects()), key=lambda r: -r[2])[:30] Log.warning("{{data}}", data=sum1) except Exception as e: Log.warning("problem in memory measure", cause=e)
def func5(): """ Модуль pympler muppy.get_objects() - метод считывает все существующтие обьекты в Python вызывая этот метод 2 раза в разный промежуток времени, мы можем выявить какие обьекты были созданы с первого среза. summary.get_diff() - выявляет какие обьекты как разницу между двумя срезами. summary.print_() - красиво выводит на экран обьекты и память ими занимаемую. """ from pympler import asizeof from pympler import muppy from pympler import summary print('\nИспользование muppy :') alll_obj_1 = muppy.get_objects() data = list(range(1000)) alll_obj_2 = muppy.get_objects() sum1 = summary.summarize(alll_obj_1) sum2 = summary.summarize(alll_obj_2) summary.print_(summary.get_diff(sum1, sum2))
def __exit__(self, exc_type, exc_val, exc_tb): if self.debug: try: gc.collect() end_memory = self.process.memory_info().rss net_memory = end_memory - self.start_memory if net_memory > 100 * 1000 * 1000: Log.warning( "MEMORY WARNING (additional {{net_memory|comma}}bytes): " + self.description, default_params=self.params, net_memory=net_memory) from pympler import summary from pympler import muppy sum1 = sorted(summary.summarize(muppy.get_objects()), key=lambda r: -r[2])[:30] Log.warning("{{data}}", data=sum1) elif end_memory > 1000 * 1000 * 1000: Log.warning( "MEMORY WARNING (over {{end_memory|comma}}bytes): " + self.description, default_params=self.params, end_memory=end_memory) from pympler import summary from pympler import muppy sum1 = sorted(summary.summarize(muppy.get_objects()), key=lambda r: -r[2])[:30] Log.warning("{{data}}", data=sum1) except Exception as e: Log.warning("problem in memory measure", cause=e)
def test_smarts_fast_reset_memory_cleanup(agent_id, seed, social_agent_scenarios): agent_type = AgentType.Buddha # Run once to initialize globals and test to see if smarts is working _memory_buildup(agent_id, seed, social_agent_scenarios, 1, None, agent_type) tr = tracker.SummaryTracker() gc.collect() initial_size = muppy.get_size(muppy.get_objects()) for _ in range(100): _memory_buildup( agent_id, seed, social_agent_scenarios, 1, None, agent_type, max_episode_steps=2, ) gc.collect() end_size = muppy.get_size(muppy.get_objects()) gc.collect() tr.print_diff() # Check for a major leak assert (end_size - initial_size < SMARTS_MEMORY_GROWTH_LIMIT ), f"End size delta {end_size - initial_size}"
def test_env_memory_cleanup(agent_id, seed, primative_scenarios): # Run once to initialize globals _, action, agent_type = (100, None, AgentType.Buddha) _env_memory_buildup(agent_id, seed, primative_scenarios, action, agent_type) gc.collect() # Memory size check size = muppy.get_size(muppy.get_objects()) gc.collect() _env_memory_buildup(agent_id, seed, primative_scenarios, action, agent_type) end_size = muppy.get_size(muppy.get_objects()) gc.collect() def success_condition(): return end_size - size < EPISODE_MEMORY_GROWTH_LIMIT if not success_condition(): # Get a diff for failure case tr = tracker.SummaryTracker() tr.print_diff() _env_memory_buildup(agent_id, seed, primative_scenarios, action, agent_type) diff = tr.diff() summary.print_(diff) diff = None gc.collect() assert success_condition(), f"Size diff {end_size - size}"
def _initialize_component(self, modeldata, namespaces, component_name, profile_memory): declaration = self.component(component_name) if component_name in modeldata._default: if declaration.type() is not Set: declaration.set_default(modeldata._default[component_name]) data = None for namespace in namespaces: if component_name in modeldata._data.get(namespace, {}): if declaration.type() is Set: data = self._tuplize( modeldata._data[namespace][component_name], declaration) else: data = modeldata._data[namespace][component_name] if not data is None: break if __debug__ and logger.isEnabledFor(logging.DEBUG): _blockName = "Model" if self.parent_block() is None \ else "Block '%s'" % self.name logger.debug("Constructing %s '%s' on %s from data=%s", declaration.__class__.__name__, declaration.name, _blockName, str(data)) try: declaration.construct(data) except: err = sys.exc_info()[1] logger.error( "Constructing component '%s' from data=%s failed:\n%s: %s", str(declaration.name), str(data).strip(), type(err).__name__, err) raise if __debug__ and logger.isEnabledFor(logging.DEBUG): _out = StringIO() declaration.pprint(ostream=_out) logger.debug("Constructed component '%s':\n%s" % (declaration.name, _out.getvalue())) if (pympler_available is True) and (profile_memory >= 2): mem_used = muppy.get_size(muppy.get_objects()) print( " Total memory = %d bytes following construction of component=%s" % (mem_used, component_name)) if (pympler_available is True) and (profile_memory >= 3): gc.collect() mem_used = muppy.get_size(muppy.get_objects()) print( " Total memory = %d bytes following construction of component=%s (after garbage collection)" % (mem_used, component_name))
def create_summary(self): """Return a summary. See also the notes on ignore_self in the class as well as the initializer documentation. """ if not self.ignore_self: res = summary.summarize(muppy.get_objects()) else: # If the user requested the data required to store summaries to be # ignored in the summaries, we need to identify all objects which # are related to each summary stored. # Thus we build a list of all objects used for summary storage as # well as a dictionary which tells us how often an object is # referenced by the summaries. # During this identification process, more objects are referenced, # namely int objects identifying referenced objects as well as the # correspondind count. # For all these objects it will be checked wether they are # referenced from outside the monitor's scope. If not, they will be # subtracted from the snapshot summary, otherwise they are # included (as this indicates that they are relevant to the # application). all_of_them = [] # every single object ref_counter = {} # how often it is referenced; (id(o), o) pairs def store_info(o): all_of_them.append(o) if id(o) in ref_counter: ref_counter[id(o)] += 1 else: ref_counter[id(o)] = 1 # store infos on every single object related to the summaries store_info(self.summaries) for k, v in self.summaries.items(): store_info(k) summary._traverse(v, store_info) # do the summary res = summary.summarize(muppy.get_objects()) # remove ids stored in the ref_counter for _id in ref_counter: # referenced in frame, ref_counter, ref_counter.keys() if len(gc.get_referrers(_id)) == (3): summary._subtract(res, _id) for o in all_of_them: # referenced in frame, summary, all_of_them if len(gc.get_referrers(o)) == (ref_counter[id(o)] + 2): summary._subtract(res, o) return res
def test_print_diff(self): """Test summary can be printed.""" try: self._stdout = sys.stdout sys.stdout = self.DevNull() sum1 = summary.summarize(muppy.get_objects()) sum2 = summary.summarize(muppy.get_objects()) sumdiff = summary.get_diff(sum1, sum2) summary.print_(sumdiff) finally: sys.stdout = self._stdout
def test_ignore_frame(self): """Test whether reference cycles are created """ gc.collect() gc.disable() objs = muppy.get_objects() del objs self.assertEqual(gc.collect(), 0) objs = muppy.get_objects(include_frames=True) del objs self.assertEqual(gc.collect(), 0) gc.enable()
def test_smarts_basic_memory_cleanup(agent_id, seed, primative_scenarios, agent_params): # Run once to initialize globals and test to see if smarts is working _memory_buildup(agent_id, seed, primative_scenarios, 100, agent_params[1], agent_params[2]) gc.collect() initial_size = muppy.get_size(muppy.get_objects()) _memory_buildup(agent_id, seed, primative_scenarios, *agent_params) end_size = muppy.get_size(muppy.get_objects()) # Check for a major leak assert (end_size - initial_size < SMARTS_MEMORY_GROWTH_LIMIT ), f"End size delta {end_size - initial_size}"
def test_print_diff(self): """Test summary can be printed.""" try: self._stdout = sys.stdout stream = StringIO() sys.stdout = stream sum1 = summary.summarize(muppy.get_objects()) sum2 = summary.summarize(muppy.get_objects()) sumdiff = summary.get_diff(sum1, sum2) summary.print_(sumdiff) self.assertIn('str', stream.getvalue()) self.assertNotIn("<class 'str", stream.getvalue()) finally: sys.stdout = self._stdout
def memusage_before_n_after(fun, *args, **kwargs): from pympler import muppy from pympler import summary from datetime import datetime before = summary.summarize(muppy.get_objects()) before_time = datetime.now() fun_ret = fun(*args, **kwargs) after_time = datetime.now() after = summary.summarize(muppy.get_objects()) diff = summary.get_diff(before, after) print "execution time: ", after_time - before_time summary.print_(diff) return fun_ret, diff
def _initialize_component(self, modeldata, namespaces, component_name, profile_memory): declaration = self.component(component_name) if component_name in modeldata._default: if declaration.type() is not Set: declaration.set_default(modeldata._default[component_name]) data = None for namespace in namespaces: if component_name in modeldata._data.get(namespace,{}): if declaration.type() is Set: data = self._tuplize(modeldata._data[namespace][component_name], declaration) else: data = modeldata._data[namespace][component_name] if not data is None: break if __debug__ and logger.isEnabledFor(logging.DEBUG): _blockName = "Model" if self.parent_block() is None \ else "Block '%s'" % self.name logger.debug( "Constructing %s '%s' on %s from data=%s", declaration.__class__.__name__, declaration.name, _blockName, str(data) ) try: declaration.construct(data) except: err = sys.exc_info()[1] logger.error( "Constructing component '%s' from data=%s failed:\n %s: %s", str(declaration.name), str(data).strip(), type(err).__name__, err ) raise if __debug__ and logger.isEnabledFor(logging.DEBUG): _out = StringIO() declaration.pprint(ostream=_out) logger.debug("Constructed component '%s':\n %s" % ( declaration.name, _out.getvalue())) if (pympler_available is True) and (profile_memory >= 2): mem_used = muppy.get_size(muppy.get_objects()) print(" Total memory = %d bytes following construction of component=%s" % (mem_used, component_name)) if (pympler_available is True) and (profile_memory >= 3): gc.collect() mem_used = muppy.get_size(muppy.get_objects()) print(" Total memory = %d bytes following construction of component=%s (after garbage collection)" % (mem_used, component_name))
def memory_summary(): from pympler import muppy, summary all_objects = muppy.get_objects() obj_summary = summary.summarize(all_objects) logger.info("\n".join(summary.format_(obj_summary)))
def dump_state(): loop = get_event_loop() print(datetime.now()) all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) summary.print_(sum1, limit=100) loop.call_later(10, dump_state)
def summary(cls): lg = InsightLogger.InsightLogger.get_logger('MemTracker', 'MemTracker.log') sm = '=====Total Memory Stats=====\n' + cls.str_print_( summary.summarize(muppy.get_objects()), limit=25) lg.info("Admin requested memory summary\n\n{}".format(sm)) return sm
def consumer(inQ, outQ): while True: try: # get a new message val = inQ.get() # this is the 'TERM' signal if val is None: break # process the data ret = f(val) if args.debug: from pympler import summary from pympler import muppy all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) print("summary:") summary.print_(sum1) from pympler import tracker tr = tracker.SummaryTracker() print("diff:") tr.print_diff() outQ.put(ret) except Exception as e: print("error!", e) break
def fileCloseEnd(cntlrWinMain, filename): if __testContext.checkMemoryOnClose: gc.collect() try: xx = cntlrWinMain.prevNumObjects cntlrWinMain.dumpIdx except: cntlrWinMain.prevNumObjects = 0 cntlrWinMain.dumpIdx = 0 cntlrWinMain.dumpIdx += 1 all_objects = muppy.get_objects() numObjects = len(all_objects) diffObjects = numObjects - cntlrWinMain.prevNumObjects cntlrWinMain.prevNumObjects = numObjects print("numObjects=" + str(numObjects) + " (" + str(diffObjects) + " more)") if False: with open(__testContext.dumpFilePrefix + str(cntlrWinMain.dumpIdx) + ".txt", "w") as text_file: idx = 0 for o in all_objects: idx += 1 otype = "" try: otype = str(type(o)) except: pass try: print("type=" + otype + " " + str(o), file=text_file) except: pass all_objects = None gc.collect() print(numObjects) print("End of close " + filename) __testContext.diffNumObjects = diffObjects
def test_smarts_episode_memory_cleanup(agent_id, seed, primative_scenarios, agent_params): MAX_EPISODE_STEPS = 100 EPISODE_COUNT = 100 STEPS_PER_YIELD = 10 _, action, agent_type = agent_params env_and_agent_spec = env_and_spec(action, agent_type, MAX_EPISODE_STEPS, primative_scenarios, seed, agent_id) size = 0 last_size = 0 gc.collect() tr = tracker.SummaryTracker() try: for current_episode in _every_nth_episode( agent_id, EPISODE_COUNT, env_and_agent_spec, steps_per_yield=STEPS_PER_YIELD): gc.collect() all_objects = muppy.get_objects() size = muppy.get_size(all_objects) tr.print_diff(summary.summarize(all_objects)) print(flush=True) all_objects = None if current_episode > STEPS_PER_YIELD: assert (size - last_size < EPISODE_MEMORY_GROWTH_LIMIT ), f"End size delta {size - last_size}" last_size = size finally: env_and_agent_spec[0].close()
def dump_objs(): global TRACKER if TRACKER is None: TRACKER = tracker.SummaryTracker() with open("obj_log.txt", "a") as fp: fp.write("Memory at {}\n".format(str(datetime.datetime.now()))) try: all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) str_sum = summary.format_(sum1) fp.write("Summary:\n") for line in str_sum: fp.write(" {}\n".format(line)) except Exception: err = traceback.format_exc() fp.write("Error: \n") fp.write(err) try: str_diff = TRACKER.format_diff() fp.write("Diff:\n") for line in str_diff: fp.write(" {}\n".format(line)) except Exception: err = traceback.format_exc() fp.write("Error: \n") fp.write(err) fp.write("\n")
def memory_profiler(self): all_objects = muppy.get_objects() stats = summary.summarize(all_objects) return { 'Memory_profiler': [l for l in summary.format_(stats, LIMIT_OBJECTS_FOR_PROFILER)] }
def test_smarts_social_agent_scenario_memory_cleanup(agent_id, seed, social_agent_scenarios, agent_type): # Run once to initialize globals and test to see if smarts is working _memory_buildup(agent_id, seed, social_agent_scenarios, 1, *agent_type) gc.collect() initial_size = muppy.get_size(muppy.get_objects()) _memory_buildup(agent_id, seed, social_agent_scenarios, 100, *agent_type) gc.collect() end_size = muppy.get_size(muppy.get_objects()) assert (end_size - initial_size < SMARTS_MEMORY_GROWTH_LIMIT ), f"End size delta {end_size - initial_size}"
def process_response(self, request, response): req = request.META['PATH_INFO'] if req.find('static') == -1 and req.find('media') == -1: print req self.end_objects = muppy.get_objects() sum_start = summary.summarize(self.start_objects) sum_end = summary.summarize(self.end_objects) diff = summary.get_diff(sum_start, sum_end) summary.print_(diff) #print '~~~~~~~~~' #cb = refbrowser.ConsoleBrowser(response, maxdepth=2, \ #str_func=output_function) #cb.print_tree() print '~~~~~~~~~' a = asizeof(response) print 'Total size of response object in kB: %s' % \ str(a / 1024.0) print '~~~~~~~~~' a = asizeof(self.end_objects) print 'Total size of end_objects in MB: %s' % \ str(a / 1048576.0) b = asizeof(self.start_objects) print 'Total size of start_objects in MB: %s' % \ str(b / 1048576.0) print '~~~~~~~~~' return response
def on_epoch_end(self, epoch, log={}): x = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss web_browser_debug = True print(x) if x > 40000: if web_browser_debug: if epoch == 0: start_in_background() tr = tracker.SummaryTracker() tr.print_diff() else: global memlist all_objects = muppy.get_objects(include_frames=True) # print(len(all_objects)) sum1 = summary.summarize(all_objects) memlist.append(sum1) summary.print_(sum1) if len(memlist) > 1: # compare with last - prints the difference per epoch diff = summary.get_diff(memlist[-2], memlist[-1]) summary.print_(diff) my_types = muppy.filter(all_objects, Type=types.ClassType) for t in my_types: print(t)
def check_leakage(): ##check for leakage all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) # Prints out a summary of the large objects summary.print_(sum1) gc.collect()
def _capture_snapshot(self): # type: () -> None """ Capture memory usage snapshot. """ capture_time = int(time.time()) # 1. Capture aggregate values all_objects = muppy.get_objects() all_objects = self._filter_muppy_objects(all_objects) sum1 = summary.summarize(all_objects) data = summary.format_(sum1, limit=50) item = { "timestamp": capture_time, "data": list(data), "type": "aggregated", } self._profiling_data.append(item) # 2. Capture diff since the last capture data = self._tracker.format_diff() item = { "timestamp": capture_time, "data": list(data), "type": "diff", } self._profiling_data.append(item)
def apply_postprocessing(data, instance=None, results=None): """ Apply post-processing steps. Required: instance: Problem instance. results: Optimization results object. """ # if not data.options.runtime.logging == 'quiet': sys.stdout.write('[%8.2f] Applying Pyomo postprocessing actions\n' % (time.time()-start_time)) sys.stdout.flush() # options are of type ConfigValue, not raw strings / atomics. for config_value in data.options.postprocess: postprocess = pyutilib.misc.import_file(config_value, clear_cache=True) if "pyomo_postprocess" in dir(postprocess): postprocess.pyomo_postprocess(data.options, instance,results) for ep in ExtensionPoint(IPyomoScriptPostprocess): ep.apply( options=data.options, instance=instance, results=results ) if (pympler_available is True) and (data.options.runtime.profile_memory >= 1): mem_used = muppy.get_size(muppy.get_objects()) if mem_used > data.local.max_memory: data.local.max_memory = mem_used print(" Total memory = %d bytes upon termination" % mem_used)
def apply_postprocessing(data, instance=None, results=None): """ Apply post-processing steps. Required: instance: Problem instance. results: Optimization results object. """ # if not data.options.runtime.logging == 'quiet': sys.stdout.write('[%8.2f] Applying Pyomo postprocessing actions\n' % (time.time() - start_time)) sys.stdout.flush() # options are of type ConfigValue, not raw strings / atomics. for config_value in data.options.postprocess: postprocess = pyutilib.misc.import_file(config_value, clear_cache=True) if "pyomo_postprocess" in dir(postprocess): postprocess.pyomo_postprocess(data.options, instance, results) for ep in ExtensionPoint(IPyomoScriptPostprocess): ep.apply(options=data.options, instance=instance, results=results) if (pympler_available is True) and (data.options.runtime.profile_memory >= 1): mem_used = muppy.get_size(muppy.get_objects()) if mem_used > data.local.max_memory: data.local.max_memory = mem_used print(" Total memory = %d bytes upon termination" % mem_used)
def printMemorySummary(self): all_objects = muppy.get_objects() out = "Total_NumObjects=" + str(len(all_objects)) + "\n" self.write(out) #filter out certain types of objects out = None types1 = muppy.filter(all_objects, Type=types.ClassType) out = "Num_Type=" + str(len(types1)) + "\n" for t in types1: out += str(t) out += "\n" self.write(out) out = None #comppare summery of memory # sumCurr = summary.summarize(all_objects) # if (self.sumPrev): # diff = summary.get_diff(sumCurr, self.sumPrev) # summary.print_(diff) # #self.write(str(diff)) # self.sumPrev = sumCurr # self.tr.print_diff() print "memory.summary.done"
def generate_loop(dump_mem): try: #raise MemoryError() class_tuple = classes if isinstance(classes, tuple) else (classes, classes) inputs = list(map(prep_in, load_classes('input/', class_tuple[0]))) outputs = list( map(prep_out, load_classes('output/', class_tuple[1]))) random.seed(1) area = size[0] * size[1] while not shutdown: results = [] metrics = [] msum = np.ones(out_channels) while len(results) < 20: n = random.randint(0, len(inputs) - 1) i = inputs[n] o = outputs[n] scale = 3.0**(0.4 - random.random()) if scale * i.shape[0] > size[0] and scale * i.shape[ 1] > size[1] and random.random() > 0.5: i = rescale(i, scale) o = rescale(o, scale) i_shape = i.shape a = -30 + 60.0 * random.random() while not test_angle(a, i_shape, size): a *= 0.5 i = rotate_f(i, a) o = rotate_f(o, a) count = int(i_shape[0] * i_shape[1] / area * 3) for p in gen_pos(a, i_shape, size, count): x, y = p[1], p[0] ip = i[y:y + size[0], x:x + size[1], :] op = o[y:y + size[0], x:x + size[1], :] if not every_flip and random.randint(0, 10) > 5: ip = np.flip(ip, 1) op = np.flip(op, 1) m = np.sum(op, axis=(0, 1)) if m.sum() == 0: continue msum += m metrics.append((len(results), m)) results.append((ip, op)) metrics = sorted(metrics, key=lambda m: -np.sum( (m[1] / msum)[:out_channels - 1])) metrics = metrics[:int(len(metrics) * 0.5)] # Reduce number of empty outputs random.shuffle(metrics) for a in metrics: r = results[a[0]] yield r if every_flip: yield np.flip(r[0], 1), np.flip(r[1], 1) if dump_mem: summary.print_(summary.summarize(muppy.get_objects())) except MemoryError: print('Memory error...') _thread.interrupt_main()
def memory_summary(): # Only import Pympler when we need it. We don't want it to # affect our process if we never call memory_summary. from pympler import summary, muppy mem_summary = summary.summarize(muppy.get_objects()) rows = summary.format_(mem_summary) return '\n'.join(rows)
def printListingUsage(self, args): all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) summary.print_(sum1) print(" ") print("Summary: ") tr = tracker.SummaryTracker() tr.print_diff()
async def log_object_summary(interval: float): from pympler import muppy, summary while True: await asyncio.sleep(interval) lines = summary.format_(summary.summarize(muppy.get_objects()), limit=20) logging.info('top objects:\n%s', '\n'.join(lines))
def _get_stats(): from pympler import muppy, summary all_objects = muppy.get_objects() result = summary.summarize(all_objects) result = result[0:20] summary = '\n'.join([l for l in summary.format_(result)]) result = '%s\n\n%s' % (summary, json.dumps(result)) return result, 200, {'content-type': 'text/plain'}
def test_smarts_repeated_runs_memory_cleanup(agent_id, seed, primative_scenarios, agent_type): # Run once to initialize globals and test to see if smarts is working _memory_buildup(agent_id, seed, primative_scenarios, 1, *agent_type) gc.collect() initial_size = muppy.get_size(muppy.get_objects()) for i in range(100): _memory_buildup(agent_id, seed, primative_scenarios, 1, *agent_type) gc.collect() end_size = muppy.get_size(muppy.get_objects()) # This "should" be roughly the same as `test_smarts_basic_memory_cleanup` assert (end_size - initial_size < SMARTS_MEMORY_GROWTH_LIMIT ), f"End size delta {end_size - initial_size}"
def memory_usage(where): """ Print out a basic summary of memory usage. """ mem_summary = summary.summarize(muppy.get_objects()) print("Memory summary:", where) summary.print_(mem_summary, limit=2) print("VM: %.2fMb" % (get_virtual_memory_usage_kb() / 1024.0))
def handle_signal_abort(self, signum, frame): Log.warn("Someone want to kill me! But I'll not die now! Hahahaha!") s = summary.summarize(muppy.get_objects()) Log.debug("Current memory usage:") summary.print_(s) diff = summary.get_diff(self.mem_sum, s) self.mem_sum = s Log.debug("New memory usage:") summary.print_(diff)
def test_untracked_containers(self): """Test whether untracked container objects are detected. """ untracked = {} tracked = {'untracked': untracked} self.assertTrue(gc.is_tracked(tracked)) self.assertFalse(gc.is_tracked(untracked)) objects = [id(o) for o in muppy.get_objects()] self.assertTrue(id(untracked) in objects)
def sig_usr(self, a, b): import threading import gc held_locks = {} code = {} curthreads = threading.enumerate() for threadId, stack in sys._current_frames().items(): name = str(threadId) for ct in curthreads: if ct.ident == threadId: name = ct.name code[name] = ["NAME: %s" % name] for filename, lineno, fname, line in traceback.extract_stack( stack): code[name].append('FILE: "%s", line %d, in %s' % (filename, lineno, fname)) if line: code[name].append(" %s" % (line.strip())) held_locks[name] = "" for lock in alllocks: if lock.writer_id == threadId: held_locks[name] += ("%s(w)" % lock.name) continue for reader_id, reader_stack in lock.reader_stacks: if reader_id == threadId: held_locks[name] += ("%s(r)" % lock.name) for k in code: log.info('\n\nLOCKS: %s \n%s' % (held_locks[k], '\n'.join(code[k]))) log.info("\n\nSTACKS:") for lock in alllocks: for (reader_id, reader_stack) in lock.reader_stacks: log.info("Lock %s (%s readers)" % (lock.name, lock.readers)) log.info("Lock reader (thread %s):" % (reader_id, )) log.info(''.join(reader_stack)) for writer_stack in lock.writer_stacks: log.info("Lock %s (%s readers)" % (lock.name, lock.readers)) log.info("Lock writer (thread %s):" % (lock.writer_id, )) log.info(''.join(writer_stack)) self.shelf.sync() gc.collect() # If we've got pympler installed, output a summary of memory usage. try: from pympler import summary, muppy summary.print_(summary.summarize(muppy.get_objects())) except: pass
def memory_summary(self): # Only import Pympler when we need it. We don't want it to # affect our process if we never call memory_summary. caller = sys._getframe(1).f_code.co_name # So we can reference the caller from pympler import summary, muppy mem_summary = summary.summarize(muppy.get_objects()) rows = summary.format_(mem_summary) indigo.server.log ('\n\nCALLED BY: ' + caller + '\n\n' + '\n'.join(rows) )
def sig_usr(self, a, b): import threading import gc held_locks = {} code = {} curthreads = threading.enumerate() for threadId, stack in sys._current_frames().items(): name = str(threadId) for ct in curthreads: if ct.ident == threadId: name = ct.name code[name] = ["NAME: %s" % name] for filename, lineno, fname, line in traceback.extract_stack(stack): code[name].append('FILE: "%s", line %d, in %s' % (filename, lineno, fname)) if line: code[name].append(" %s" % (line.strip())) held_locks[name] = "" for lock in alllocks: if lock.writer_id == threadId: held_locks[name] += ("%s(w)" % lock.name) continue for reader_id, reader_stack in lock.reader_stacks: if reader_id == threadId: held_locks[name] += ("%s(r)" % lock.name) for k in code: log.info('\n\nLOCKS: %s \n%s' % (held_locks[k], '\n'.join(code[k]))) log.info("\n\nSTACKS:") for lock in alllocks: for (reader_id, reader_stack) in lock.reader_stacks: log.info("Lock %s (%s readers)" % (lock.name, lock.readers)) log.info("Lock reader (thread %s):" % (reader_id,)) log.info(''.join(reader_stack)) for writer_stack in lock.writer_stacks: log.info("Lock %s (%s readers)" % (lock.name, lock.readers)) log.info("Lock writer (thread %s):" % (lock.writer_id,)) log.info(''.join(writer_stack)) self.shelf.sync() gc.collect() # If we've got pympler installed, output a summary of memory usage. try: from pympler import summary, muppy from pympler.asizeof import asizeof summary.print_(summary.summarize(muppy.get_objects())) except: pass
def profile_expose_method(profiled_method_wrapper, accept, args, func, kw, exclude_from_memory_profiling): """ Targeted to profile a specific method that wraps HTTP request processing endpoints into database context. :param profiled_method_wrapper: method wrapped around profiled call to be passed in to memory profiler :param accept: param specific to profiled call :param args: args of a function that is being wrapped by a profiled method :param func: function that is being wrapped by a profiled method :param kw: kwargs of a function that is being wrapped by a profiled method :return: output of a profiled method without modification """ if not exclude_from_memory_profiling and get_memory_profile_logging_on() and \ check_memory_profile_package_wide_disable(func): controller_class = args[0].__class__.__name__ if args and len(args) > 0 else '' end_point_name_parts = [s for s in [func.__module__, controller_class, func.__name__] if s != ''] end_point_name = ".".join(end_point_name_parts) is_pympler_on = _is_pympler_profiling_value_on(end_point_name) profile_output = {'output': {}} if is_pympler_on: all_objects = muppy.get_objects() all_objects_summary_before = summary.summarize(all_objects) memory_profile = memory_usage((_profile_me, (profile_output, profiled_method_wrapper, func, accept, args, kw), {}), interval=0.1) output = profile_output['output'] if is_pympler_on: all_objects_summary_after = summary.summarize(all_objects) diff = summary.get_diff(all_objects_summary_before, all_objects_summary_after) diff_less = summary.format_(diff) diff_out = '' for s in diff_less: diff_out += s+'\n' thread_log.info("================ PYMPLER OUTPUT <{}> ==============\n{}".format(end_point_name, diff_out)) try: message = json.dumps({'log_type': 'memory_profile', 'proc_id': os.getpid(), 'name': func.__name__, 'module': func.__module__, 'mem_profile': memory_profile, 'min': min(memory_profile), 'max': max(memory_profile), 'diff': max(memory_profile) - min(memory_profile), 'leaked': memory_profile[-1] - memory_profile[0], 'args': [arg for arg in args[1:]], # exclude self 'kwargs': kw}) memory_log.info(message, extra={'controller_module': func.__module__, 'controller_class': controller_class, 'endpoint': func.__name__}) except Exception as e: thread_log.exception('Logger failed: {}'.format(e)) else: output = profiled_method_wrapper(accept, args, func, kw) return output
def memory_summary(self, summarize=True): "Using pympler summarize module to view memory summary." if summarize: all_objects = muppy.get_objects() Logger.info("ENV: \nMemory Footprint:") Logger.info("-----------------") return summary.print_(summary.summarize(all_objects), limit=50) else: Logger.info("ENV: \nMemory Tracker:") Logger.info("---------------") self.mem_tracker.print_diff()
def get_report(self): all_objects = muppy.get_objects() size = get_size(all_objects) report = summary.summarize(all_objects) sort_index = self.cleaned_data['sort_by'] limit = self.cleaned_data['limit'] report.sort(key=lambda item: item[sort_index], reverse=True) if limit: report = report[:limit] return size, report
def print_memory(count=30): ''' Print the statistics of the objects in the memory. Need pympler to use. ''' from pympler import muppy, summary gc.collect() all_objects = muppy.get_objects() my_types = muppy.filter(all_objects, Type=wx.Object) sum1 = summary.summarize(my_types) # sum1 = summary.summarize(all_objects) summary.print_(sum1, limit=count)
def __init__(self, **kwargs): super(GlobalContainer, self).__init__(**kwargs) self._keyboard = None self.request_keyboard() self.working_dir = './' self.tutorial = None self.popup_stack = [] window.Window.bind(on_resize=self.on_resize) if DEBUG: self.tracker = ClassTracker() self.tracker.track_object(MenuButton) self.all_objects = muppy.get_objects()
def __init__(self, ignore_self=True): """Constructor. The number of summaries managed by the tracker has an performance impact on new summaries, iff you decide to exclude them from further summaries. Therefore it is suggested to use them economically. Keyword arguments: ignore_self -- summaries managed by this object will be ignored. """ self.s0 = summary.summarize(muppy.get_objects()) self.summaries = {} self.ignore_self = ignore_self
def print_muppy_sumary(): # http://pythonhosted.org/Pympler/index.html try: from pympler import muppy, summary except ImportError: print("WARNING: pympler not installed") return # from pympler.classtracker import ClassTracker # from pympler.classtracker_stats import HtmlStats global all_objects, obj_summary, class_tracker if all_objects is None: all_objects = muppy.get_objects() obj_summary = summary.summarize(all_objects) summary.print_(obj_summary) # class_tracker = ClassTracker() # class_tracker.track_class(FICSPlayer, trace=1) # class_tracker.track_class(ICGameModel, resolution_level=2, trace=1) else: obj_summary2 = summary.summarize(muppy.get_objects()) diff = summary.get_diff(obj_summary, obj_summary2) summary.print_(diff, limit=200)
def request(ctx, flow): global step, ssl print("==========") print("GC: {}".format(gc.collect())) print("Threads: {}".format(threading.active_count())) step += 1 if step == 1: all_objects = muppy.get_objects() ssl = muppy.filter(all_objects, SSL.Connection)[0] if step == 2: ib = refbrowser.InteractiveBrowser(ssl, 2, str_fun, repeat=False) del ssl # do this to unpollute view ib.main(True)
def analyzeAllMFCC(): client = MongoClient() db = client.audiograins grainEntries = db.grains query = grainEntries.find({ "mfcc00" : { "$exists": False }}) print("Analyzing MFCC for " + str(query.count()) + " grains") for grain in tqdm(query): mfccs = analyzeMFCC(grain) for mfccIndex in range(0, len(mfccs)): update = {"mfcc" + format(mfccIndex, '02') : mfccs[mfccIndex]} grainEntries.update_one({"_id": grain["_id"]}, {"$set" : update}) summary.print_(summary.summarize(muppy.get_objects())) client.close()
def muppy_loop(self): """Generator method for looping over the iterations and writing out the muppy output.""" # Loop over the desired number of iterations. for i in range(self.num): # Muppy output, only output at every 100th iteration. if not i % 100: self.file.write("Iteration %i\n" % i) self.file.write("Muppy heap:\n") for line in muppy.summary.format_(muppy.summary.summarize(muppy.get_objects())): self.file.write("%s\n" % line) self.file.write("\n\n\n") self.file.flush() # Yield the loop index. yield i
def collect_and_dump_root(self): log.msg('Profiling memory for OmsRoot objects...', system=self.__name__) try: import inspect from sys import getsizeof from BTrees.OOBTree import OOBucket from ZEO.Exceptions import ClientDisconnected from opennode.oms.model.model.root import OmsRoot data = [] all_objects = muppy.get_objects() roots = muppy.filter(all_objects, Type=OmsRoot) logger.info('Root profile follows (%s rows)' % len(roots)) gc.collect() for ue in roots: referrers = [] for ref in gc.get_referrers(ue): try: if inspect.isframe(ref): continue # local object ref elif isinstance(ref, list): referrers.append('list len=%s id=%x' % (len(ref), id(ref))) elif isinstance(ref, OOBucket): referrers.append('OOBucket len=%s id=%x' % (len(ref), id(ref))) else: sref = repr(ref) referrers.append(sref) except ClientDisconnected: referrers.append('ClientDisconnected') data.append((referrers, str(ue), repr(ue), str(getsizeof(ue)))) rrows = [('object', 'raw', 'size', 'referrers')] + data rows = _format_table(rrows) for row in rows: logger.info(row) log.msg('Profiling Omsroot memory done', system=self.__name__) del all_objects gc.collect() return defer.succeed(None) except Exception, e: import traceback logger.error(traceback.format_exc(e)) return defer.fail(None)
def test(self): tr = tracker.SummaryTracker() testDir = os.path.dirname(os.path.abspath(sys.modules[__name__].__file__)) testFileSmall = testDir + "/solvency/2.0/random/spv_20_instance.xbrl" logFile = testDir + "/tmp/test.log" dumpFilePrefix = testDir + "/tmp/dump_" prevNumObjects = 0 for idx in range(3): # increase this range for testing print("\nIteration " + str(idx)) arelleRunArgs = ['--keepOpen', '--logFile', 'logToStdErr', '--logfile', logFile, '--file', testFileSmall] cntlr = parseAndRun(arelleRunArgs) cntlr.modelManager.close() cntlr.close() del cntlr gc.collect() all_objects = muppy.get_objects() numObjects = len(all_objects) diffObjects = numObjects - prevNumObjects prevNumObjects = numObjects print(str(numObjects) + " (" + str(diffObjects) + " more)") browserRoot = None if False: # <<<--- set this to get object dump file with open(dumpFilePrefix + str(idx) + ".txt", "w") as text_file: idx = 0 for o in all_objects: if browserRoot is None and isinstance(o, arelle.ModelValue.QName): browserRoot = o idx += 1 otype = "" try: otype = str(type(o)) except: pass try: print("type=" + otype + " " + str(o), file=text_file) except: pass if False: ibrowser = refbrowser.InteractiveBrowser(browserRoot) ibrowser.main() all_objects= None gc.collect() tr.print_diff() if idx > 1: assert diffObjects < 50, "Check for new objects leak"
def getDebugInfo(self, itemname): """Give debug info about a particular item.""" global profile outf=StringIO() if itemname == "": outf.write("the item was empty") if profile: all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) summary.print_(sum1, 100) ib = refbrowser.InteractiveBrowser(self) ib.main() return outf.getvalue() itemname=keywords.fixID(itemname) itemlist=vtype.parseItemList(itemname) item=self.getSubValue(itemlist) item.writeDebug(outf) return outf.getvalue()
def profile(self, frame, event, arg): # arg req to match signature """Profiling method used to profile matching codepoints and events.""" if (self.events is None) or (event in self.events): frame_info = inspect.getframeinfo(frame) cp = (frame_info[0], frame_info[2], frame_info[1]) if self.codepoint_included(cp): objects = muppy.get_objects() size = muppy.get_size(objects) if cp not in self.memories: self.memories[cp] = [0, 0, 0, 0] self.memories[cp][0] = 1 self.memories[cp][1] = size self.memories[cp][2] = size else: self.memories[cp][0] += 1 if self.memories[cp][1] > size: self.memories[cp][1] = size if self.memories[cp][2] < size: self.memories[cp][2] = size