def func5(): """ Модуль pympler muppy.get_objects() - метод считывает все существующтие обьекты в Python вызывая этот метод 2 раза в разный промежуток времени, мы можем выявить какие обьекты были созданы с первого среза. summary.get_diff() - выявляет какие обьекты как разницу между двумя срезами. summary.print_() - красиво выводит на экран обьекты и память ими занимаемую. """ from pympler import asizeof from pympler import muppy from pympler import summary print('\nИспользование muppy :') alll_obj_1 = muppy.get_objects() data = list(range(1000)) alll_obj_2 = muppy.get_objects() sum1 = summary.summarize(alll_obj_1) sum2 = summary.summarize(alll_obj_2) summary.print_(summary.get_diff(sum1, sum2))
def check_leakage(): ##check for leakage all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) # Prints out a summary of the large objects summary.print_(sum1) gc.collect()
def generate_loop(dump_mem): try: #raise MemoryError() class_tuple = classes if isinstance(classes, tuple) else (classes, classes) inputs = list(map(prep_in, load_classes('input/', class_tuple[0]))) outputs = list( map(prep_out, load_classes('output/', class_tuple[1]))) random.seed(1) area = size[0] * size[1] while not shutdown: results = [] metrics = [] msum = np.ones(out_channels) while len(results) < 20: n = random.randint(0, len(inputs) - 1) i = inputs[n] o = outputs[n] scale = 3.0**(0.4 - random.random()) if scale * i.shape[0] > size[0] and scale * i.shape[ 1] > size[1] and random.random() > 0.5: i = rescale(i, scale) o = rescale(o, scale) i_shape = i.shape a = -30 + 60.0 * random.random() while not test_angle(a, i_shape, size): a *= 0.5 i = rotate_f(i, a) o = rotate_f(o, a) count = int(i_shape[0] * i_shape[1] / area * 3) for p in gen_pos(a, i_shape, size, count): x, y = p[1], p[0] ip = i[y:y + size[0], x:x + size[1], :] op = o[y:y + size[0], x:x + size[1], :] if not every_flip and random.randint(0, 10) > 5: ip = np.flip(ip, 1) op = np.flip(op, 1) m = np.sum(op, axis=(0, 1)) if m.sum() == 0: continue msum += m metrics.append((len(results), m)) results.append((ip, op)) metrics = sorted(metrics, key=lambda m: -np.sum( (m[1] / msum)[:out_channels - 1])) metrics = metrics[:int(len(metrics) * 0.5)] # Reduce number of empty outputs random.shuffle(metrics) for a in metrics: r = results[a[0]] yield r if every_flip: yield np.flip(r[0], 1), np.flip(r[1], 1) if dump_mem: summary.print_(summary.summarize(muppy.get_objects())) except MemoryError: print('Memory error...') _thread.interrupt_main()
def on_epoch_end(self, epoch, log={}): x = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss web_browser_debug = True print(x) if x > 40000: if web_browser_debug: if epoch == 0: start_in_background() tr = tracker.SummaryTracker() tr.print_diff() else: global memlist all_objects = muppy.get_objects(include_frames=True) # print(len(all_objects)) sum1 = summary.summarize(all_objects) memlist.append(sum1) summary.print_(sum1) if len(memlist) > 1: # compare with last - prints the difference per epoch diff = summary.get_diff(memlist[-2], memlist[-1]) summary.print_(diff) my_types = muppy.filter(all_objects, Type=types.ClassType) for t in my_types: print(t)
def file_test(rows=500000, cols=50): "File test" print("Creating file with {} rows and {} columns".format(rows, cols)) file = create_file(rows, cols) print("Size of the file: {:.2f} MiB".format(getsize(file) / (1024 * 1024))) print("Reading file") sum1 = summarize(get_objects()) las = read(file) sum2 = summarize(get_objects()) diff = get_diff(sum1, sum2) print_(diff) for curve in las.curves: print("Name: {}, Min: {:.2f}, Mean: {:.2f}, Max: {:.2f}" .format(curve.mnemonic, nanmin(curve.data), nanmean(curve.data), nanmax(curve.data))) del las las = read(file) del las las = read(file) del las las = read(file) del las print("Happy end")
def test_env_memory_cleanup(agent_id, seed, primative_scenarios): # Run once to initialize globals _, action, agent_type = (100, None, AgentType.Buddha) _env_memory_buildup(agent_id, seed, primative_scenarios, action, agent_type) gc.collect() # Memory size check size = muppy.get_size(muppy.get_objects()) gc.collect() _env_memory_buildup(agent_id, seed, primative_scenarios, action, agent_type) end_size = muppy.get_size(muppy.get_objects()) gc.collect() def success_condition(): return end_size - size < EPISODE_MEMORY_GROWTH_LIMIT if not success_condition(): # Get a diff for failure case tr = tracker.SummaryTracker() tr.print_diff() _env_memory_buildup(agent_id, seed, primative_scenarios, action, agent_type) diff = tr.diff() summary.print_(diff) diff = None gc.collect() assert success_condition(), f"Size diff {end_size - size}"
def dump_state(): loop = get_event_loop() print(datetime.now()) all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) summary.print_(sum1, limit=100) loop.call_later(10, dump_state)
def process_response(self, request, response): req = request.META['PATH_INFO'] if req.find('static') == -1 and req.find('media') == -1: print req self.end_objects = muppy.get_objects() sum_start = summary.summarize(self.start_objects) sum_end = summary.summarize(self.end_objects) diff = summary.get_diff(sum_start, sum_end) summary.print_(diff) #print '~~~~~~~~~' #cb = refbrowser.ConsoleBrowser(response, maxdepth=2, \ #str_func=output_function) #cb.print_tree() print '~~~~~~~~~' a = asizeof(response) print 'Total size of response object in kB: %s' % \ str(a / 1024.0) print '~~~~~~~~~' a = asizeof(self.end_objects) print 'Total size of end_objects in MB: %s' % \ str(a / 1048576.0) b = asizeof(self.start_objects) print 'Total size of start_objects in MB: %s' % \ str(b / 1048576.0) print '~~~~~~~~~' return response
def consumer(inQ, outQ): while True: try: # get a new message val = inQ.get() # this is the 'TERM' signal if val is None: break # process the data ret = f(val) if args.debug: from pympler import summary from pympler import muppy all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) print("summary:") summary.print_(sum1) from pympler import tracker tr = tracker.SummaryTracker() print("diff:") tr.print_diff() outQ.put(ret) except Exception as e: print("error!", e) break
def memory_usage(where): """ Print out a basic summary of memory usage. """ mem_summary = summary.summarize(muppy.get_objects()) print("Memory summary:", where) summary.print_(mem_summary, limit=2) print("VM: %.2fMb" % (get_virtual_memory_usage_kb() / 1024.0))
def printListingUsage(self, args): all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) summary.print_(sum1) print(" ") print("Summary: ") tr = tracker.SummaryTracker() tr.print_diff()
def print_diff(self, summary1=None, summary2=None): """Compute diff between to summaries and print it. If no summary is provided, the diff from the last to the current summary is used. If summary1 is provided the diff from summary1 to the current summary is used. If summary1 and summary2 are provided, the diff between these two is used. """ summary.print_(self.diff(summary1=summary1, summary2=summary2))
def handle_signal_abort(self, signum, frame): Log.warn("Someone want to kill me! But I'll not die now! Hahahaha!") s = summary.summarize(muppy.get_objects()) Log.debug("Current memory usage:") summary.print_(s) diff = summary.get_diff(self.mem_sum, s) self.mem_sum = s Log.debug("New memory usage:") summary.print_(diff)
def sig_usr(self, a, b): import threading import gc held_locks = {} code = {} curthreads = threading.enumerate() for threadId, stack in sys._current_frames().items(): name = str(threadId) for ct in curthreads: if ct.ident == threadId: name = ct.name code[name] = ["NAME: %s" % name] for filename, lineno, fname, line in traceback.extract_stack( stack): code[name].append('FILE: "%s", line %d, in %s' % (filename, lineno, fname)) if line: code[name].append(" %s" % (line.strip())) held_locks[name] = "" for lock in alllocks: if lock.writer_id == threadId: held_locks[name] += ("%s(w)" % lock.name) continue for reader_id, reader_stack in lock.reader_stacks: if reader_id == threadId: held_locks[name] += ("%s(r)" % lock.name) for k in code: log.info('\n\nLOCKS: %s \n%s' % (held_locks[k], '\n'.join(code[k]))) log.info("\n\nSTACKS:") for lock in alllocks: for (reader_id, reader_stack) in lock.reader_stacks: log.info("Lock %s (%s readers)" % (lock.name, lock.readers)) log.info("Lock reader (thread %s):" % (reader_id, )) log.info(''.join(reader_stack)) for writer_stack in lock.writer_stacks: log.info("Lock %s (%s readers)" % (lock.name, lock.readers)) log.info("Lock writer (thread %s):" % (lock.writer_id, )) log.info(''.join(writer_stack)) self.shelf.sync() gc.collect() # If we've got pympler installed, output a summary of memory usage. try: from pympler import summary, muppy summary.print_(summary.summarize(muppy.get_objects())) except: pass
def memusage(line): limit = int(line or 10) memory_info = psutil.Process().memory_info() print("VMS {:.2f}M, RSS {:.2f}M, SHR {:.2f}M, objects:".format( float(memory_info.vms) / 1024.0 / 1024.0, float(memory_info.rss) / 1024.0 / 1024.0, float(memory_info.shared) / 1024.0 / 1024.0)) mem_summary = summary.summarize(muppy.get_objects()) summary.print_(mem_summary, limit=limit)
def sig_usr(self, a, b): import threading import gc held_locks = {} code = {} curthreads = threading.enumerate() for threadId, stack in sys._current_frames().items(): name = str(threadId) for ct in curthreads: if ct.ident == threadId: name = ct.name code[name] = ["NAME: %s" % name] for filename, lineno, fname, line in traceback.extract_stack(stack): code[name].append('FILE: "%s", line %d, in %s' % (filename, lineno, fname)) if line: code[name].append(" %s" % (line.strip())) held_locks[name] = "" for lock in alllocks: if lock.writer_id == threadId: held_locks[name] += ("%s(w)" % lock.name) continue for reader_id, reader_stack in lock.reader_stacks: if reader_id == threadId: held_locks[name] += ("%s(r)" % lock.name) for k in code: log.info('\n\nLOCKS: %s \n%s' % (held_locks[k], '\n'.join(code[k]))) log.info("\n\nSTACKS:") for lock in alllocks: for (reader_id, reader_stack) in lock.reader_stacks: log.info("Lock %s (%s readers)" % (lock.name, lock.readers)) log.info("Lock reader (thread %s):" % (reader_id,)) log.info(''.join(reader_stack)) for writer_stack in lock.writer_stacks: log.info("Lock %s (%s readers)" % (lock.name, lock.readers)) log.info("Lock writer (thread %s):" % (lock.writer_id,)) log.info(''.join(writer_stack)) self.shelf.sync() gc.collect() # If we've got pympler installed, output a summary of memory usage. try: from pympler import summary, muppy from pympler.asizeof import asizeof summary.print_(summary.summarize(muppy.get_objects())) except: pass
def memory_profile_print(): from pympler import muppy, summary all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) # Prints out a summary of the large objects summary.print_(sum1) # Get references to certain types of objects such as dataframe dataframes = [ao for ao in all_objects if isinstance(ao, pd.DataFrame)] for d in dataframes: print(d.columns.values) print(len(d))
def test_print_diff(self): """Test summary can be printed.""" try: self._stdout = sys.stdout sys.stdout = self.DevNull() sum1 = summary.summarize(muppy.get_objects()) sum2 = summary.summarize(muppy.get_objects()) sumdiff = summary.get_diff(sum1, sum2) summary.print_(sumdiff) finally: sys.stdout = self._stdout
def run_history_cleaner(): gc.collect() all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) # Prints out a summary of the large objects summary.print_(sum1) remove_files(".png") remove_files(".rcg") threading.Timer(60, run_history_cleaner).start()
def dosomething(): print "memory total" all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) summary.print_(sum1) print "memory difference" tr.print_diff() # ........ 爬取任务 pass
def print_memory(count=30): ''' Print the statistics of the objects in the memory. Need pympler to use. ''' from pympler import muppy, summary gc.collect() all_objects = muppy.get_objects() my_types = muppy.filter(all_objects, Type=wx.Object) sum1 = summary.summarize(my_types) # sum1 = summary.summarize(all_objects) summary.print_(sum1, limit=count)
def profile(diff=False): # type: (bool) -> None """A simple profiler using stuff from pympler""" if _DO_PROFILE and _MEM_PROFILE: if diff: TRACKER.print_diff() else: summary.print_(summary.summarize(muppy.get_objects())) elif _DO_PROFILE and not _MEM_PROFILE: logging.error( "Could not find 'pympler' module, please install with pip and run again" ) else: pass
def analyze_memory(tracker): all_objects = muppy.get_objects() num = len(all_objects) print 'number of objects:', num sum1 = summary.summarize(all_objects) print 'sumary of all objects' summary.print_(sum1) print 'difference: ' tracker.print_diff() pdb.set_trace()
def dosomething(): print("memory total") all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) summary.print_(sum1) print("memory difference") tr.print_diff() # ........ 爬取任务 a = {} for i in range(10000): a[i] = i time.sleep(0.5) return
def memory_details(task, method=None): if method == 'pympler': from pympler import muppy, summary all_objs = muppy.get_objects() summary.print_(summary.summarize(all_objs)) elif method == 'mem_top': from mem_top import mem_top task.log(mem_top()) else: import subprocess result = subprocess.check_output( 'ps --no-headers -eo pmem,vsize,rss,pid,cmd | sort -k 1 -nr', shell=True) task.log('\n' + result.decode('utf8'))
def test_print_diff(self): """Test summary can be printed.""" try: self._stdout = sys.stdout stream = StringIO() sys.stdout = stream sum1 = summary.summarize(muppy.get_objects()) sum2 = summary.summarize(muppy.get_objects()) sumdiff = summary.get_diff(sum1, sum2) summary.print_(sumdiff) self.assertIn('str', stream.getvalue()) self.assertNotIn("<class 'str", stream.getvalue()) finally: sys.stdout = self._stdout
def print_diff(self, ignore=[]): """Print the diff to the last time the state of objects was measured. keyword arguments ignore -- list of objects to ignore """ # ignore this and the caller frame ignore.append(inspect.currentframe()) #PYCHOK change ignore diff = self.get_diff(ignore) print("Added objects:") summary.print_(summary.summarize(diff['+'])) print("Removed objects:") summary.print_(summary.summarize(diff['-'])) # manual cleanup, see comment above del ignore[:]
def memusage_before_n_after(fun, *args, **kwargs): from pympler import muppy from pympler import summary from datetime import datetime before = summary.summarize(muppy.get_objects()) before_time = datetime.now() fun_ret = fun(*args, **kwargs) after_time = datetime.now() after = summary.summarize(muppy.get_objects()) diff = summary.get_diff(before, after) print "execution time: ", after_time - before_time summary.print_(diff) return fun_ret, diff
def analyzeAllMFCC(): client = MongoClient() db = client.audiograins grainEntries = db.grains query = grainEntries.find({ "mfcc00" : { "$exists": False }}) print("Analyzing MFCC for " + str(query.count()) + " grains") for grain in tqdm(query): mfccs = analyzeMFCC(grain) for mfccIndex in range(0, len(mfccs)): update = {"mfcc" + format(mfccIndex, '02') : mfccs[mfccIndex]} grainEntries.update_one({"_id": grain["_id"]}, {"$set" : update}) summary.print_(summary.summarize(muppy.get_objects())) client.close()
def finished_level(self, level, name): if level <= int(self.reporting_level): self.check_for_leaks_before_next_test = True if level > int(self.reporting_level): return if self.report_delta: color = self.REPORT_DETAILS[level].color report = u'Memory Delta Report for %s: %s\n' % ( self.REPORT_DETAILS[level].title.upper(), name) memory_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss print('Peak memory usage for %s: %s' % (name, memory_usage)) old_summary = self.previous_summaries[level] if not self.current_summary: self.current_summary = self.get_summary() diff = self._fast_get_summary_diff(old_summary, self.current_summary) filtered_diff = [row for row in diff if row[1] or row[2]] if filtered_diff: print(termcolor.colored(report, color)) print(summary.print_(filtered_diff)) else: report += 'No changes\n' report += 'Peak memory usage: %s' % memory_usage print(termcolor.colored(report, color)) self.previous_summaries[level] = self.current_summary
def epoch(model, train_seq, val_seq, noaction=False, step=16): MEMPROFILE = False import gc for j in range(len(train_seq)): try: (_, _), klasses, _, buildings = train_seq[j] except Exception as exc: logger.error(str(exc)) continue #for (pre,post), klasses, mask, buildings in train_seq: buildings = np.array(buildings) klasses = np.array(klasses) for i in range(0, len(buildings), step): if i + step > len(buildings): if noaction: buf1, buf2 = buildings[i:], klasses[i:] logger.info( f"{j}:{i}: {len(buf1)} samples accessed successfully.") continue model.fit(buildings[i:], klasses[i:], verbose=2, shuffle=False, use_multiprocessing=False) if noaction: buf1, buf2 = buildings[i:i + step], klasses[i:i + step] logger.info( f"{j}:{i}: {len(buf1)} samples accessed successfully.") continue model.fit(buildings[i:i + step], klasses[i:i + step], verbose=2, shuffle=False, use_multiprocessing=False) if j % 100 == 0: num_uncollectable = gc.collect(2) logger.info("Uncollectable objects: %d" % num_uncollectable) if MEMPROFILE is True and j > 100: from pympler import muppy, summary objs = muppy.get_objects() sum = summary.summarize(objs) summary.print_(sum) sys.exit()
def collection_post(self): print("REQUEST {}".format(self.request.json)) sums = [] for key in 'sum1', 'sum2': sums.append(self.request.json[key]) with transaction.manager: query = self.request.dbsession.query(ObjectSummary) query = query.filter(ObjectSummary.id.in_(sums)) query = query.order_by(ObjectSummary.created) dbsums = query.all() sums = [m.content for m in dbsums] diff = summary.get_diff(*sums) output = io.StringIO() with contextlib.redirect_stdout(output): summary.print_(diff) # import pdb ; pdb.set_trace() return dict(result='testing', output=output.getvalue())
def getDebugInfo(self, itemname): """Give debug info about a particular item.""" global profile outf=StringIO() if itemname == "": outf.write("the item was empty") if profile: all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) summary.print_(sum1, 100) ib = refbrowser.InteractiveBrowser(self) ib.main() return outf.getvalue() itemname=keywords.fixID(itemname) itemlist=vtype.parseItemList(itemname) item=self.getSubValue(itemlist) item.writeDebug(outf) return outf.getvalue()
async def seememory(self, ctx: Context): ''' Shows you the number of each created object in the program ''' # summary.print_(summary.summarize(muppy.get_objects())) awaitable_dot_code = self.bot.loop.run_in_executor( None, muppy.get_objects) try: objects = await wait_for(awaitable_dot_code, timeout=10.0, loop=self.bot.loop) except Exception as e: await ctx.send(f"{e!s}") return summary.print_(summary.summarize(objects)) await ctx.send("Printed to console.")
def getDebugInfo(self, itemname): """Give debug info about a particular item.""" global profile outf = StringIO() if itemname == "": outf.write("the item was empty") if profile: all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) summary.print_(sum1, 100) ib = refbrowser.InteractiveBrowser(self) ib.main() return outf.getvalue() itemname = keywords.fixID(itemname) itemlist = vtype.parseItemList(itemname) item = self.getSubValue(itemlist) item.writeDebug(outf) return outf.getvalue()
def check_memory(): logger.debug('Checking memory.') logger.debug(objgraph.by_type('SpiderTask')) logger.debug(objgraph.by_type('TaskExecutor')) logger.debug(objgraph.by_type('Future')) logger.debug(objgraph.by_type('PeriodicCallback')) logger.debug(objgraph.by_type('Workspace')) logger.debug(objgraph.by_type('MultipartRequestBodyProducer')) logger.debug(objgraph.by_type('HTTPRequest')) future_objs = objgraph.by_type('Future') if future_objs: objgraph.show_chain( objgraph.find_backref_chain(future_objs[-1], objgraph.is_proper_module), filename='chain.png' ) all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) # Prints out a summary of the large objects summary.print_(sum1)
def memory_summary(self, summarize=True): "Using pympler summarize module to view memory summary." if summarize: all_objects = muppy.get_objects() Logger.info("ENV: \nMemory Footprint:") Logger.info("-----------------") return summary.print_(summary.summarize(all_objects), limit=50) else: Logger.info("ENV: \nMemory Tracker:") Logger.info("---------------") self.mem_tracker.print_diff()
async def pympler_summary(request: web.Request) -> web.StreamResponse: """ Build and return a Pympler summary: https://pympler.readthedocs.io/en/latest/muppy.html#the-summary-module Example: curl -v -X POST 'localhost:9999/pympler/summary' """ try: from pympler import muppy, summary except ImportError: return web.Response(status=500, text='pympler not installed\n') log = request.app[_LOG] all_objects = muppy.get_objects() sum_ = summary.summarize(all_objects) if 'print' in request.query: log.info('pympler.summary.print') summary.print_(sum_) return web.Response(status=204) else: log.info('pympler.summary') return web.json_response(sum_)
def print_muppy_sumary(): # http://pythonhosted.org/Pympler/index.html try: from pympler import muppy, summary except ImportError: print("WARNING: pympler not installed") return # from pympler.classtracker import ClassTracker # from pympler.classtracker_stats import HtmlStats global all_objects, obj_summary, class_tracker if all_objects is None: all_objects = muppy.get_objects() obj_summary = summary.summarize(all_objects) summary.print_(obj_summary) # class_tracker = ClassTracker() # class_tracker.track_class(FICSPlayer, trace=1) # class_tracker.track_class(ICGameModel, resolution_level=2, trace=1) else: obj_summary2 = summary.summarize(muppy.get_objects()) diff = summary.get_diff(obj_summary, obj_summary2) summary.print_(diff, limit=200)
def create_instance(self, filename=None, data=None, name=None, namespace=None, namespaces=None, preprocess=False, profile_memory=0, report_timing=False, clone=None): """ Create a concrete instance of an abstract model, possibly using data read in from a file. Optional: filename: The name of a Pyomo Data File that will be used to load data into the model. data: A dictionary containing initialization data for the model to be used if there is no filename name: The name given to the model. namespace: A namespace used to select data. namespaces: A list of namespaces used to select data. preprocess: If True, then preprocess the constructed model. profile_memory: A number that indicates the profiling level. report_timing: Report timing statistics during construction. clone: Force a clone of the model if this is True. """ if self._constructed: logger.warning("DEPRECATION WARNING: Cannot call Model.create_instance() on a concrete model.") return self if name is None: name = self.name if not filename is None: data = filename if data is None: data = {} # # Generate a warning if this is a concrete model but the filename is specified. # A concrete model is already constructed, so passing in a data file is a waste # of time. # if self.is_constructed() and isinstance(filename, basestring): msg = "The filename=%s will not be loaded - supplied as an argument to the create_instance() method of a ConcreteModel instance with name=%s." % (filename, name) logger.warning(msg) # # If construction is deferred, then clone the model and # if not self._constructed: instance = self.clone() if namespaces is None or len(namespaces) == 0: instance.load(data, namespaces=[None], profile_memory=profile_memory, report_timing=report_timing) else: instance.load(data, namespaces=list(namespaces)+[None], profile_memory=profile_memory, report_timing=report_timing) else: if clone: instance = self.clone() else: instance = self # # Preprocess the new model # if preprocess is True: print(" Model preprocessing during construction has been deprecated.") if False and preprocess is True: if report_timing is True: start_time = time.time() instance.preprocess() if report_timing is True: total_time = time.time() - start_time print(" %6.2f seconds required for preprocessing" % total_time) if (pympler_available is True) and (profile_memory >= 2): mem_used = muppy.get_size(muppy.get_objects()) print(" Total memory = %d bytes following instance preprocessing" % mem_used) print("") if (pympler_available is True) and (profile_memory >= 2): print("") print(" Summary of objects following instance preprocessing") post_preprocessing_summary = summary.summarize(muppy.get_objects()) summary.print_(post_preprocessing_summary, limit=100) if not name is None: instance.name=name # # Indicate that the model is concrete/constructed # instance._constructed = True return instance
longest_anagrams = None # tuple 2 words for w in words: key = get_histogram(w) if key not in histograms: histograms[key] = [w] else: histograms[key].append(w) if longest_anagrams is None or len(w) > len(longest_anagrams[0]): longest_anagrams = histograms[key] return longest_anagrams from pympler import summary, muppy if __name__ == "__main__": print('SOWPODS analyzer') words = read_swopods() print('Total: %s words' % len(words)) print('%s is the longest palindrome' % (find_longest_palindrome(words))) char, frequency, word = find_most_repeated_char(words) print("'%s' is the most repeated character in one word (in '%s', occurs %s times)" % (char, word, frequency)) print("%s are the longest anagrams" % (find_longest_anagram(words))) all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) summary.print_(sum1)
def before_after_each_function(request): global _global_collect_info import psutil current_pids = set(proc.pid for proc in psutil.process_iter()) before_curr_proc_memory_info = psutil.Process().memory_info() if _global_collect_info and DEBUG_MEMORY_INFO: try: from pympler import summary, muppy sum1 = summary.summarize(muppy.get_objects()) except: pydev_log.exception() sys.stdout.write( ''' =============================================================================== Memory before: %s %s =============================================================================== ''' % (request.function, format_memory_info(psutil.virtual_memory(), before_curr_proc_memory_info))) yield processes_info = [] for proc in psutil.process_iter(): if proc.pid not in current_pids: try: processes_info.append( 'New Process: %s(%s) - %s' % ( proc.name(), proc.pid, format_process_memory_info(proc.memory_info()) ) ) except psutil.NoSuchProcess: pass # The process could've died in the meanwhile after_curr_proc_memory_info = psutil.Process().memory_info() if DEBUG_MEMORY_INFO: try: if after_curr_proc_memory_info.rss - before_curr_proc_memory_info.rss > 10 * 1000 * 1000: # 10 MB leak if _global_collect_info: sum2 = summary.summarize(muppy.get_objects()) diff = summary.get_diff(sum1, sum2) sys.stdout.write('===============================================================================\n') sys.stdout.write('Leak info:\n') sys.stdout.write('===============================================================================\n') summary.print_(diff) sys.stdout.write('===============================================================================\n') _global_collect_info = True # We'll only really collect the info on the next test (i.e.: if at one test # we used too much memory, the next one will start collecting) else: _global_collect_info = False except: pydev_log.exception() sys.stdout.write( ''' =============================================================================== Memory after: %s %s%s =============================================================================== ''' % ( request.function, format_memory_info(psutil.virtual_memory(), after_curr_proc_memory_info), '' if not processes_info else '\nLeaked processes:\n' + '\n'.join(processes_info)), )
def main(): adv_messages = rawdata.load_dict_from_json(ADV_MESSAGES_PATH) banned_users = [] while True: accounts = rawdata.load_dict_from_json(ACCOUNTS_PATH) users = init(accounts) accounts.clear() try: queue = Queue() users_signed = [] for user in users: if not user in banned_users: queue.put(user) for i in range(len(users) - len(banned_users)): thread = Thread(target=signing, args=(queue, users_signed, banned_users)) thread.daemon = True thread.start() queue.join() del users[:] users = users_signed.copy() del users_signed[:] queue = Queue() for user in users: if not user in banned_users: queue.put(user) theard_count = 3 if len(users) >= 5 else 1 for i in range(theard_count): thread = Thread(target=group_creating_desire_c, args=(queue, COUNT_OF_GROUPS, NAME_OF_GROUPS)) thread.daemon = True thread.start() queue.join() arg_list = {"counter":0, "adv_counter":0, "iteration":0} while True: queue = Queue() del_targ = [] targets = rawdata.load_dict_from_json(TARGETS_PATH) len_targets = len(targets) time_start = time.time() for user in users: if not user in banned_users: queue.put(user) print("незабаненных пользователей %s/%s | забаненны %s" % (len(users) - len(banned_users), len(users), banned_users)) for i in range(len(users) - len(banned_users)): thread = Thread(target=posting_for_all_users, args=(queue, targets, adv_messages, arg_list, del_targ, banned_users)) thread.daemon = True thread.start() queue.join() time_stop = time.time() time_dif = time_stop - time_start wait_time = 60*(24 + random.uniform(0, 5)) - time_dif u_banned_users = [user.username for user in banned_users] rawdata.delete_from_json(data=del_targ, path_to_data=TARGETS_PATH) rawdata.delete_from_json(data=u_banned_users, path_to_data=ACCOUNTS_PATH) del del_targ[:] targets.clear() del targets gc.collect() summary.print_(summary.summarize(muppy.get_objects())) print("Осталось целей %s" % len_targets) print("Всего/рекламных %s/%s за %s мин" % (arg_list["counter"], arg_list["adv_counter"], str(time_dif / 60))) if wait_time > 0: print("Пауза %s минут\n" % str(wait_time / 60)) time.sleep(wait_time) else: print("Продолжаем без паузы %s мин запаздывания\n" % str(-1 * wait_time / 60)) arg_list["iteration"] += 1 except: print("Ошибка в posting.py/main %s" % sys.exc_info()[0]) raise
def process_response(self, request, response): req = request.META['PATH_INFO'] if not self.is_ignored(req): if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' '~~~~~~~~~~~~~~' ) print '\n\n' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info(u'REQUESTED URL: {}'.format(req)) print u'REQUESTED URL: {}'.format(req) self.end_objects = muppy.get_objects() if SHOW['request_summary']: sum_start = summary.summarize(self.start_objects) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~ SUMMARIZE REQUEST OBJECTS ~~~~~~~~~') for row in sorted( sum_start, key=lambda i: i[2], reverse=True)[:15]: logger.info( "type: %60s , # objects: %10d, total size: %s", *(row[0], row[1], filesizeformat(row[2])) ) print '~~~~~~~~~ SUMMARIZE REQUEST OBJECTS ~~~~~~~~~' summary.print_(sum_start) if SHOW['response_summary']: sum_end = summary.summarize(self.end_objects) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~ SUMMARIZE RESPONSE OBJECTS ~~~~~~~~~') for row in sorted( sum_end, key=lambda i: i[2], reverse=True)[:15]: logger.info( "type: %60s , # objects: %10d, total size: %s", *(row[0], row[1], filesizeformat(row[2])) ) print '~~~~~~~~~ SUMMARIZE RESPONSE OBJECTS ~~~~~~~~~' summary.print_(sum_end) if SHOW['compared_request_response_summaries']: diff = summary.get_diff(sum_start, sum_end) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~ COMPARED REQUEST & RESPONSE SUMMARIES ' '~~~~~~~~~' ) for row in sorted( diff, key=lambda i: i[2], reverse=True)[:15]: logger.info( "type: %60s , # objects: %10d, total size: %s", *(row[0], row[1], filesizeformat(row[2])) ) print \ '~~~~~~~~~ COMPARED REQUEST & RESPONSE SUMMARIES ~~~~~~~~~' summary.print_(diff) # print '~~~~~~~~~' # cb = refbrowser.ConsoleBrowser( # response, maxdepth=2, str_func=output_function) # cb.print_tree() a = asizeof(response) a_string = 'Total size of response object in kB: %s' % \ str(a/1024.0) b = asizeof(self.end_objects) b_string = 'Total size of end_objects in MB: %s' % str(b/1048576.0) c = asizeof(self.start_objects) c_string = 'Total size of start_objects in MB: %s' % \ str(c/1048576.0) if SHOW_ON_DJANGO_DEBUG_TOOLBAR_LOGGIN_PANEL: logger.info( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' '~~~~~~~~~~~~~~' ) logger.info(a_string) logger.info(b_string) logger.info(c_string) logger.info( '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' '~~~~~~~~~~~~~~' ) print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' print a_string print b_string print c_string print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\ '~~~~~~~~~~~~~~~~~~~~~~~~~~' return response
def _load_model_data(self, modeldata, namespaces, **kwds): """ Load declarations from a DataPortal object. """ # # As we are primarily generating objects here (and acyclic ones # at that), there is no need to run the GC until the entire # model is created. Simple reference-counting should be # sufficient to keep memory use under control. # with PauseGC() as pgc: # # Unlike the standard method in the pympler summary # module, the tracker doesn't print 0-byte entries to pad # out the limit. # profile_memory = kwds.get('profile_memory', 0) if (pympler_available is True) and (profile_memory >= 2): mem_used = muppy.get_size(muppy.get_objects()) print("") print(" Total memory = %d bytes prior to model " "construction" % mem_used) if (pympler_available is True) and (profile_memory >= 3): gc.collect() mem_used = muppy.get_size(muppy.get_objects()) print(" Total memory = %d bytes prior to model " "construction (after garbage collection)" % mem_used) # # Do some error checking # for namespace in namespaces: if not namespace is None and not namespace in modeldata._data: msg = "Cannot access undefined namespace: '%s'" raise IOError(msg % namespace) # # Initialize each component in order. # for component_name, component in iteritems(self.component_map()): if component.type() is Model: continue self._initialize_component(modeldata, namespaces, component_name, profile_memory) if False: total_time = time.time() - start_time if isinstance(component, IndexedComponent): clen = len(component) else: assert isinstance(component, Component) clen = 1 print(" %%6.%df seconds required to construct component=%s; %d indicies total" \ % (total_time>=0.005 and 2 or 0, component_name, clen) \ % total_time) tmp_clone_counter = expr_common.clone_counter if clone_counter != tmp_clone_counter: clone_counter = tmp_clone_counter print(" Cloning detected! (clone count: %d)" % clone_counters) # Note: As is, connectors are expanded when using command-line pyomo but not calling model.create(...) in a Python script. # John says this has to do with extension points which are called from commandline but not when writing scripts. # Uncommenting the next two lines switches this (command-line fails because it tries to expand connectors twice) #connector_expander = ConnectorExpander() #connector_expander.apply(instance=self) if (pympler_available is True) and (profile_memory >= 2): print("") print(" Summary of objects following instance construction") post_construction_summary = summary.summarize(muppy.get_objects()) summary.print_(post_construction_summary, limit=100) print("")
def create_instance( self, filename=None, data=None, name=None, namespace=None, namespaces=None, profile_memory=0, report_timing=False, **kwds ): """ Create a concrete instance of an abstract model, possibly using data read in from a file. Parameters ---------- filename: `str`, optional The name of a Pyomo Data File that will be used to load data into the model. data: `dict`, optional A dictionary containing initialization data for the model to be used if there is no filename name: `str`, optional The name given to the model. namespace: `str`, optional A namespace used to select data. namespaces: `list`, optional A list of namespaces used to select data. profile_memory: `int`, optional A number that indicates the profiling level. report_timing: `bool`, optional Report timing statistics during construction. """ # # Generate a warning if this is a concrete model but the # filename is specified. A concrete model is already # constructed, so passing in a data file is a waste of time. # if self.is_constructed() and isinstance(filename, string_types): msg = "The filename=%s will not be loaded - supplied as an " \ "argument to the create_instance() method of a "\ "concrete instance with name=%s." % (filename, name) logger.warning(msg) if 'clone' in kwds: kwds.pop('clone') deprecation_warning( "Model.create_instance() no longer accepts the 'clone' " "argument: the base abstract model is always cloned.") if 'preprocess' in kwds: kwds.pop('preprocess') deprecation_warning( "Model.create_instance() no longer accepts the preprocess' " "argument: preprocessing is always deferred to when the " "model is sent to the solver") if kwds: msg = \ """Model.create_instance() passed the following unrecognized keyword arguments (which have been ignored):""" for k in kwds: msg = msg + "\n '%s'" % (k,) logger.error(msg) if self.is_constructed(): deprecation_warning( "Cannot call Model.create_instance() on a constructed " "model; returning a clone of the current model instance.") return self.clone() if report_timing: pyomo.common.timing.report_timing() if name is None: name = self.name if filename is not None: if data is not None: logger.warning("Model.create_instance() passed both 'filename' " "and 'data' keyword arguments. Ignoring the " "'data' argument") data = filename if data is None: data = {} # # Clone the model and load the data # instance = self.clone() if name is not None: instance._name = name # If someone passed a rule for creating the instance, fire the # rule before constructing the components. if instance._rule is not None: instance._rule(instance) if namespaces: _namespaces = list(namespaces) else: _namespaces = [] if namespace is not None: _namespaces.append(namespace) if None not in _namespaces: _namespaces.append(None) instance.load( data, namespaces=_namespaces, profile_memory=profile_memory ) # # Preprocess the new model # if False and preprocess is True: if report_timing is True: start_time = time.time() instance.preprocess() if report_timing is True: total_time = time.time() - start_time print(" %6.2f seconds required for preprocessing" % total_time) if (pympler_available is True) and (profile_memory >= 2): mem_used = muppy.get_size(muppy.get_objects()) print(" Total memory = %d bytes following instance preprocessing" % mem_used) print("") if (pympler_available is True) and (profile_memory >= 2): print("") print(" Summary of objects following instance preprocessing") post_preprocessing_summary = summary.summarize(muppy.get_objects()) summary.print_(post_preprocessing_summary, limit=100) # # Indicate that the model is concrete/constructed # instance._constructed = True # # Change this class from "Abstract" to "Concrete". It is # absolutely crazy that this is allowed in Python, but since the # AbstractModel and ConcreteModel are basically identical, we # can "reassign" the new concrete instance to be an instance of # ConcreteModel # instance.__class__ = ConcreteModel return instance
def log_mem(mes=''): if options.debug: all_objects = muppy.get_objects() sum1 = summary.summarize(all_objects) app_log.log(logging.DEBUG, 'mem (%s): %d' % (mes, len(all_objects))) summary.print_(sum1)
def memory_usage(where): """Print out a basic summary of memory usage.""" mem_summary = summary.summarize(muppy.get_objects()) log.debug("Memory summary: {}".format(where)) summary.print_(mem_summary, limit=2) log.debug("VM: {:2f}Mb".format(get_virtual_memory_usage_kb() / 1024.0))
def create_model(data): """ Create instance of Pyomo model. Return: model: Model object. instance: Problem instance. symbol_map: Symbol map created when writing model to a file. filename: Filename that a model instance was written to. """ # if not data.options.runtime.logging == 'quiet': sys.stdout.write('[%8.2f] Creating model\n' % (time.time()-start_time)) sys.stdout.flush() # if (pympler_available is True) and (data.options.runtime.profile_memory >= 1): global memory_data mem_used = muppy.get_size(muppy.get_objects()) data.local.max_memory = mem_used print(" Total memory = %d bytes prior to model construction" % mem_used) # # Create Model # ep = ExtensionPoint(IPyomoScriptCreateModel) model_name = 'model' if data.options.model.object_name is not None: model_name = data.options.model.object_name if model_name in dir(data.local.usermodel): if len(ep) > 0: msg = "Model construction function 'create_model' defined in " \ "file '%s', but model is already constructed!" raise SystemExit(msg % data.options.model.filename) model = getattr(data.local.usermodel, data.options.model.object_name) if model is None: msg = "'%s' object is 'None' in module %s" raise SystemExit(msg % (model_name, data.options.model.filename)) else: if len(ep) == 0: msg = "Neither '%s' nor 'pyomo_create_model' are available in " \ 'module %s' raise SystemExit(msg % ( model_name, data.options.model.filename )) elif len(ep) > 1: msg = 'Multiple model construction plugins have been registered!' raise SystemExit(msg) else: model_options = data.options.model.options.value() #if model_options is None: #model_options = [] model = ep.service().apply( options = pyutilib.misc.Container(*data.options), model_options=pyutilib.misc.Container(*model_options) ) # for ep in ExtensionPoint(IPyomoScriptPrintModel): ep.apply( options=data.options, model=model ) # # Create Problem Instance # ep = ExtensionPoint(IPyomoScriptCreateDataPortal) if len(ep) > 1: msg = 'Multiple model data construction plugins have been registered!' raise SystemExit(msg) if len(ep) == 1: modeldata = ep.service().apply( options=data.options, model=model ) else: modeldata = DataPortal() if model._constructed: # # TODO: use a better test for ConcreteModel # instance = model elif len(data.options.data.files) > 1: # # Load a list of *.dat files # for file in data.options.data.files: suffix = (file).split(".")[-1] if suffix != "dat": msg = 'When specifiying multiple data files, they must all ' \ 'be *.dat files. File specified: %s' raise SystemExit(msg % str( file )) modeldata.load(filename=file, model=model) instance = model.create_instance(modeldata, namespaces=data.options.data.namespaces, profile_memory=data.options.runtime.profile_memory, report_timing=data.options.runtime.report_timing) elif len(data.options.data.files) == 1: # # Load a *.dat file or process a *.py data file # suffix = (data.options.data.files[0]).split(".")[-1].lower() if suffix == "dat": instance = model.create_instance(data.options.data.files[0], namespaces=data.options.data.namespaces, profile_memory=data.options.runtime.profile_memory, report_timing=data.options.runtime.report_timing) elif suffix == "py": userdata = pyutilib.misc.import_file(data.options.data.files[0], clear_cache=True) if "modeldata" in dir(userdata): if len(ep) == 1: msg = "Cannot apply 'pyomo_create_modeldata' and use the" \ " 'modeldata' object that is provided in the model" raise SystemExit(msg) if userdata.modeldata is None: msg = "'modeldata' object is 'None' in module %s" raise SystemExit(msg % str( data.options.data.files[0] )) modeldata=userdata.modeldata else: if len(ep) == 0: msg = "Neither 'modeldata' nor 'pyomo_create_dataportal' " \ 'is defined in module %s' raise SystemExit(msg % str( data.options.data.files[0] )) modeldata.read(model) instance = model.create_instance(modeldata, namespaces=data.options.data.namespaces, profile_memory=data.options.runtime.profile_memory, report_timing=data.options.runtime.report_timing) elif suffix == "yml" or suffix == 'yaml': try: import yaml except: msg = "Cannot apply load data from a YAML file: PyYaml is not installed" raise SystemExit(msg) modeldata = yaml.load(open(data.options.data.files[0])) instance = model.create_instance(modeldata, namespaces=data.options.data.namespaces, profile_memory=data.options.runtime.profile_memory, report_timing=data.options.runtime.report_timing) else: raise ValueError("Unknown data file type: "+data.options.data.files[0]) else: instance = model.create_instance(modeldata, namespaces=data.options.data.namespaces, profile_memory=data.options.runtime.profile_memory, report_timing=data.options.runtime.report_timing) # modify_start_time = time.time() for ep in ExtensionPoint(IPyomoScriptModifyInstance): if data.options.runtime.report_timing is True: tick = time.time() ep.apply( options=data.options, model=model, instance=instance ) if data.options.runtime.report_timing is True: print(" %6.2f seconds to apply %s" % (time.time() - tick, type(ep))) tick = time.time() # for transformation in data.options.transform: with TransformationFactory(transformation) as xfrm: instance = xfrm.create_using(instance) if instance is None: raise SystemExit("Unexpected error while applying " "transformation '%s'" % transformation) # if data.options.runtime.report_timing is True: total_time = time.time() - modify_start_time print(" %6.2f seconds required for problem transformations" % total_time) if logger.isEnabledFor(logging.DEBUG): print("MODEL INSTANCE") instance.pprint() print("") for ep in ExtensionPoint(IPyomoScriptPrintInstance): ep.apply( options=data.options, instance=instance ) fname=None smap_id=None if not data.options.model.save_file is None: if data.options.runtime.report_timing is True: write_start_time = time.time() if data.options.model.save_file == True: if data.local.model_format in (ProblemFormat.cpxlp, ProblemFormat.lpxlp): fname = (data.options.data.files[0])[:-3]+'lp' else: fname = (data.options.data.files[0])[:-3]+str(data.local.model_format) format=data.local.model_format else: fname = data.options.model.save_file format= data.options.model.save_format io_options = {} if data.options.model.symbolic_solver_labels: io_options['symbolic_solver_labels'] = True if data.options.model.file_determinism != 1: io_options['file_determinism'] = data.options.model.file_determinism (fname, smap_id) = instance.write(filename=fname, format=format, io_options=io_options) if not data.options.runtime.logging == 'quiet': if not os.path.exists(fname): print("ERROR: file "+fname+" has not been created!") else: print("Model written to file '"+str(fname)+"'") if data.options.runtime.report_timing is True: total_time = time.time() - write_start_time print(" %6.2f seconds required to write file" % total_time) if (pympler_available is True) and (data.options.runtime.profile_memory >= 2): print("") print(" Summary of objects following file output") post_file_output_summary = summary.summarize(muppy.get_objects()) summary.print_(post_file_output_summary, limit=100) print("") for ep in ExtensionPoint(IPyomoScriptSaveInstance): ep.apply( options=data.options, instance=instance ) if (pympler_available is True) and (data.options.runtime.profile_memory >= 1): mem_used = muppy.get_size(muppy.get_objects()) if mem_used > data.local.max_memory: data.local.max_memory = mem_used print(" Total memory = %d bytes following Pyomo instance creation" % mem_used) return pyutilib.misc.Options( model=model, instance=instance, smap_id=smap_id, filename=fname, local=data.local )
def summarize_memory(): print("Virtual machine: {:.2f}Mb".format( psutil.Process().memory_info_ex().vms / (1024 * 1024))) summary.print_(summary.summarize(muppy.get_objects()), limit=1)
def memusage(o): summary.print_( summary.summarize(o))
def print_summary(): """Print a summary of all known objects.""" summary.print_(summary.summarize(get_objects()))
import random import hashlib from pympler import summary, muppy, asizeof from time import time hs = set() q = 1000000 def rand_string(l): return ''.join(chr(random.randint(97,122)) for i in range(l)) x = 1 ti = time() for i in range(q): fields = (rand_string(8) for i in range(12)) md = hashlib.md5() md.update('*'.join(fields)) h = md.digest() hs.add(h) if x % 100000 == 0: print x print time() - ti ti = time() x +=1 all_objects = muppy.get_objects() suml = summary.summarize(all_objects) summary.print_(suml) len(hs)