def checkpoint_memory(): '''This test is meant to be run manually, since it depends on memory_profiler and its behavior may vary.''' try: from memory_profiler import memory_usage except ImportError: return def f(a): for _ in range(10): a = np.sin(a**2 + 1) return a checkpointed_f = checkpoint(f) def testfun(f, x): for _ in range(5): x = f(x) return np.sum(x) gradfun = grad(testfun, 1) A = npr.RandomState(0).randn(100000) max_usage = max(memory_usage((gradfun, (f, A)))) max_checkpointed_usage = max(memory_usage((gradfun, (checkpointed_f, A)))) assert max_checkpointed_usage < max_usage / 2.
def assertNotIncreasingMemory(self, f, num_iters=100000, increase_threshold_absolute_mb=10): """Assert memory usage doesn't increase beyond given threshold for f.""" with context.eager_mode(): # Warm up. f() # Wait for background threads to start up and take over memory. # FIXME: The nature of this test leaves few other options. Maybe there # is a better way to do this. time.sleep(4) initial = memory_profiler.memory_usage(-1)[0] for _ in six.moves.range(num_iters): f() increase = memory_profiler.memory_usage(-1)[0] - initial assert increase < increase_threshold_absolute_mb, ( "Increase is too high. Initial memory usage: %f MB. Increase: %f MB. " "Maximum allowed increase: %f") % (initial, increase, increase_threshold_absolute_mb)
def trace_calls_and_returns(frame, event, arg): global just_returned co = frame.f_code func_name = co.co_name if just_returned: just_returned = False print 'just_returned' print 'mem before gc ' + str(memory_usage(-1)) # gc.collect() print 'mem after gc ' + str(memory_usage(-1)) if func_name == 'memory_usage': return if func_name == 'write': # Ignore write() calls from print statements return line_no = frame.f_lineno filename = co.co_filename if event == 'call': print 'Call to %s on line %s of %s' % (func_name, line_no, filename) return trace_calls_and_returns elif event == 'return': print 'return of %s => %s' % (func_name, arg) just_returned = True print 'mem before gc ' + str(memory_usage(-1)) # gc.collect() print 'mem after gc ' + str(memory_usage(-1)) elif event == 'line': print 'a line event ' + str(line_no) return
def memory_profile(function, *args, **kwargs): gc.collect() baseline = memory_profiler.memory_usage()[0] max_usage = memory_profiler.memory_usage( (function, args, kwargs), max_usage=True, ) return max_usage[0] - baseline
def run_tests_if_main(measure_mem=False): """Run tests in a given file if it is run as a script.""" local_vars = inspect.currentframe().f_back.f_locals if not local_vars.get('__name__', '') == '__main__': return # we are in a "__main__" try: import faulthandler faulthandler.enable() except Exception: pass with warnings.catch_warnings(record=True): # memory_usage internal dep. mem = int(round(max(memory_usage(-1)))) if measure_mem else -1 if mem >= 0: print('Memory consumption after import: %s' % mem) t0 = time.time() peak_mem, peak_name = mem, 'import' max_elapsed, elapsed_name = 0, 'N/A' count = 0 for name in sorted(list(local_vars.keys()), key=lambda x: x.lower()): val = local_vars[name] if name.startswith('_'): continue elif callable(val) and name.startswith('test'): count += 1 doc = val.__doc__.strip() if val.__doc__ else name sys.stdout.write('%s ... ' % doc) sys.stdout.flush() try: t1 = time.time() if measure_mem: with warnings.catch_warnings(record=True): # dep warn mem = int(round(max(memory_usage((val, (), {}))))) else: val() mem = -1 if mem >= peak_mem: peak_mem, peak_name = mem, name mem = (', mem: %s MB' % mem) if mem >= 0 else '' elapsed = int(round(time.time() - t1)) if elapsed >= max_elapsed: max_elapsed, elapsed_name = elapsed, name sys.stdout.write('time: %0.3f sec%s\n' % (elapsed, mem)) sys.stdout.flush() except Exception as err: if 'skiptest' in err.__class__.__name__.lower(): sys.stdout.write('SKIP (%s)\n' % str(err)) sys.stdout.flush() else: raise elapsed = int(round(time.time() - t0)) sys.stdout.write('Total: %s tests\n• %0.3f sec (%0.3f sec for %s)\n• ' 'Peak memory %s MB (%s)\n' % (count, elapsed, max_elapsed, elapsed_name, peak_mem, peak_name))
def test_memory_leak(): """https://github.com/emcconville/wand/pull/127""" minimum = 1.0 with Color('NONE') as nil_color: minimum = ctypes.sizeof(nil_color.raw) consumes = memory_usage((color_memory_leak, (), {})) assert consumes[-1] - consumes[0] <= minimum
def custom_minimize(function, algorithm, bounds = None, guess = None): def iter_minimize(): # lightweight version of iter_minimize for a single optimization method start = timeit.default_timer() result = 0 # some minimization techniques do not require an initial guess if guess is not None: result = algorithm(function, guess) else: result = algorithm(function, bounds) iterations = -1 if 'nit' in result.keys(): iterations = result.get('nit') stop = timeit.default_timer() return iterations, start, stop, result #tracks amount of memory used by current process (-1) every interval (.2 seconds) current_memory = memory_usage(-1, interval=.2) most_mem = np.append(most_mem, max(current_memory)) num_iters, start, stop, result = iter_minimize() exec_time = stop-start print '{0} took {1} seconds. The result, {2} was found at ({3})'.format(algorithm.__name__,exec_time,result.fun,result.x) print '{0} used {1} megabytes and took {2} iterations'.format(algorithm.__name__,most_mem,num_iters) print
def during_execution_memory_sampler(): import time import memory_profiler global keep_watching, peak_memory_usage peak_memory_usage = -1 keep_watching = True n = 0 WAIT_BETWEEN_SAMPLES_SECS = 0.001 MAX_ITERATIONS = 60.0 / WAIT_BETWEEN_SAMPLES_SECS while True: mem_usage = memory_profiler.memory_usage()[0] peak_memory_usage = max(mem_usage, peak_memory_usage) time.sleep(WAIT_BETWEEN_SAMPLES_SECS) if not keep_watching or n > MAX_ITERATIONS: # exit if we've been told our command has finished or if it has run # for more than a sane amount of time (e.g. maybe something crashed # and we don't want this to carry on running) if n > MAX_ITERATIONS: print( "{} SOMETHING WEIRD HAPPENED AND THIS RAN FOR TOO LONG, THIS THREAD IS KILLING ITSELF".format( __file__ ) ) break n += 1
def test(prefix, item_ct, loop_ct): h5 = h5py.File(sys.argv[1], "a") cc = h5.id.get_mdc_config() print(cc.max_size) print(h5.id.get_access_plist().get_cache()) cc.min_size = 1024 * 1024 cc.max_size = 1024 * 1024 h5.id.set_mdc_config(cc) # h5.create_group(b'\xff') # force new compact-or-indexed group with high byte # del h5[b'\xff'] print("start %s %.3f" % (prefix, time.time() - start)) path_i = 0 for i in range(loop_ct): data = np.zeros(item_ct, dtype=np.float32) i_str = "%05d" % i # path = '' # path = '/'.join((prefix, i_str[0], i_str[1], i_str[2], i_str[3])) path = str(path_i) # print(path + '/' + str(i)) ds = h5.create_dataset(path + "/" + str(i), data=data) # ds = h5.create_dataset(path + '/' + str(i), (item_ct,), dtype=np.float32) # ds[:] = data # h5.create_group(path + '/' + i_str) if i != 0 and i % close_interval == 0: path_i += 1 # print(i, '%.1f' % (time.time() - start), h5.id.get_mdc_size(), h5.id.get_mdc_hit_rate()) print( i, "%.1f" % (time.time() - start), h5.id.get_mdc_size(), h5.id.get_mdc_hit_rate(), memory_profiler.memory_usage(-1), ) print("end %s %.3f" % (prefix, time.time() - start)) h5.close()
def __init__(self, method, n=1, timeout=120.0): from memory_profiler import memory_usage import time from util import avg self.timeout = timeout max_memory_list = [] avg_memory_list = [] time_list = [] for i in range(n): runner = self._Runner(memory_usage, {'proc': method, 'interval': 0.01, 'retval': True}, timeout) baseline = avg(memory_usage((lambda: None, ()))) # baseline memory usage measurement start_time = time.time() runner.run() # actual running of the method end_time = time.time() if runner.result is not None: mem_use, self.result = runner.result memory = map(lambda x: x - baseline, mem_use) else: # there was a timeout import sys memory = [-sys.maxint] self.result = None max_memory_list.append(max(memory)) avg_memory_list.append(avg(memory)) time_list.append((end_time - start_time) * 1000) self.max_memory = avg(max_memory_list) self.avg_memory = avg(avg_memory_list) self.time = avg(time_list)
def run(self): jobs = Job.objects.due() if self.do_profile: try: import memory_profiler prof_string = "[%8.2f MB] " % memory_profiler.memory_usage()[0] except ImportError: prof_string = "No profiler found" else: prof_string = "" if jobs: logging.info( "%sRunning %d due jobs... (%s)" % (prof_string, jobs.count(), ", ".join(['"%s"' % job.name for job in jobs])) ) for job in Job.objects.due(): job.run() else: logging.debug("%sNo jobs due to run." % prof_string) if self.do_gc: gc.collect()
def time_svd(svdfunc, N1, N2, f, rseed=0, bestof=3, args=None, matfunc=np.asarray, **kwargs): if args is None: args = () N1_N2_f = np.broadcast(N1, N2, f) times = [] memory = [] for (N1, N2, f) in N1_N2_f: M = sparse_matrix(N1, N2, f, matfunc, rseed) t_best = np.inf mem_best = np.inf for i in range(bestof): t0 = time() if args: _args = [M] _args.extend(list(args)) else: _args = (M,) mem_usage = max(memory_usage((svdfunc, _args, kwargs))) t1 = time() t_best = min(t_best, t1 - t0) mem_best = min(mem_best, mem_usage) times.append(t_best) memory.append(mem_best) return np.array(times).reshape(N1_N2_f.shape), np.array(memory).reshape(N1_N2_f.shape)
def run_example(self, theInput, theOutput): # Import program (decrapted in 3.4, no other way at the moment) from importlib.machinery import SourceFileLoader solution = SourceFileLoader("solution", self.programPath).load_module() # Feed the input with PatchStd(theInput) as std: # Start time counter startTime = time.time() # Run the program solution.main() # Get end time endTime = time.time() - startTime # Get memory (include current tests ~14MB but more or less is that) mem = memory_usage(max_usage=True) # Check output actual_output = std.getStdOut().getvalue() self.assertEqual(actual_output, theOutput) # Print time (not do before because output is not yet retrieved) std.restore() print("\tTime: %.3f sec" % endTime) print("\tMemory: %.3f MB" % mem) # Show errors if any errors = std.getStdErr().getvalue() if errors != '': print("\t" + errors)
def watch_memory(): import time # bring in the global memory usage value from the previous iteration global previous_call_memory_usage, peak_memory_usage, keep_watching nbr_commands = len(In) new_memory_usage = memory_profiler.memory_usage()[0] memory_delta = new_memory_usage - previous_call_memory_usage keep_watching = False peaked_memory_usage = max(0, peak_memory_usage - new_memory_usage) # calculate time delta using global t1 (from the pre-run event) and current # time time_delta_secs = time.time() - t1 cmd = In[nbr_commands - 1] # convert the results into a pretty string output_template = "'{cmd}' used {memory_delta:0.4f} MiB RAM in {time_delta:0.2f}s, peaked {peaked_memory_usage:0.2f} MiB above current, total RAM usage {memory_usage:0.2f} MiB" output = output_template.format( time_delta=time_delta_secs, cmd=cmd, memory_delta=memory_delta, peaked_memory_usage=peaked_memory_usage, memory_usage=new_memory_usage, ) print(str(output)) previous_call_memory_usage = new_memory_usage
def f(a, v): print "running", os.getpid(), len(a) print "in process", memory_profiler.memory_usage() #a[0] = 'x' print "did arr get changed in process?", a[:5] v.value = len(a) time.sleep(5)
def memorySummary(): global memLog global memList global memMax if memLog: gc.collect() try: from memory_profile import memory_usage mem = memory_usage(-1)[0] except: raise # except: # try: # mem = int(os.popen('ps -p %d -o vsz|tail -1' % os.getpid()).read())/1024.0 # except: # try: # #CLE doesn't have full ps functionality # #try to find first occurence of pid, then get memory usage slot # #mwf debug # #import pdb # #pdb.set_trace() # plist = os.popen('ps ').read() # pr = plist.split('%s' % os.getpid()) # #print 'pr = %s ' % pr # #print 'pr[1] = %s ' % pr[1] # mem = int(pr[1].split()[1])/1024.0 # except: # logEvent("memory function doesn't work on this platform\n") # mem = 0 if mem > 0: for pair in memList: logEvent(`pair[0]`+" %"+`100.0*pair[1]/memMax`)
def watch_memory(): # bring in the global memory usage value from the previous iteration global previous_call_memory_usage, peak_memory_usage, keep_watching, \ watching_memory, input_cells new_memory_usage = memory_profiler.memory_usage()[0] memory_delta = new_memory_usage - previous_call_memory_usage keep_watching = False peaked_memory_usage = max(0, peak_memory_usage - new_memory_usage) # calculate time delta using global t1 (from the pre-run event) and current # time time_delta_secs = time.time() - t1 num_commands = len(input_cells) - 1 cmd = "In [{}]".format(num_commands) # convert the results into a pretty string output_template = ("{cmd} used {memory_delta:0.4f} MiB RAM in " "{time_delta:0.2f}s, peaked {peaked_memory_usage:0.2f} " "MiB above current, total RAM usage " "{memory_usage:0.2f} MiB") output = output_template.format(time_delta=time_delta_secs, cmd=cmd, memory_delta=memory_delta, peaked_memory_usage=peaked_memory_usage, memory_usage=new_memory_usage) if watching_memory: print(str(output)) previous_call_memory_usage = new_memory_usage
def watch_memory(self): if not self.watching_memory: return # calculate time delta using global t1 (from the pre-run # event) and current time self.time_delta = time.time() - self.t1 new_memory_usage = memory_profiler.memory_usage()[0] self.memory_delta = new_memory_usage - self.previous_call_memory_usage self.keep_watching = False self.peaked_memory_usage = max(0, self.peak_memory_usage - new_memory_usage) num_commands = len(self.input_cells) - 1 cmd = "In [{}]".format(num_commands) # convert the results into a pretty string output_template = ("{cmd} used {memory_delta:0.3f} MiB RAM in " "{time_delta:0.3f}s, peaked {peaked_memory_usage:0.3f} " "MiB above current, total RAM usage " "{memory_usage:0.3f} MiB") output = output_template.format( time_delta=self.time_delta, cmd=cmd, memory_delta=self.memory_delta, peaked_memory_usage=self.peaked_memory_usage, memory_usage=new_memory_usage) print(str(output)) self.previous_call_memory_usage = new_memory_usage
def leak(): data = [pa.array(np.concatenate([np.random.randn(100000)] * 1000))] table = pa.Table.from_arrays(data, ['foo']) while True: print('calling to_pandas') print('memory_usage: {0}'.format(memory_profiler.memory_usage())) table.to_pandas() gc.collect()
def Map(L): results = {} # key value storage for line in L: key = str(line[0] +":" + line[1] +":" + line[2]); try: results[key] += 1 except KeyError: results[key] = 1 return results, [multiprocessing.current_process().name, memory_usage(-1, interval=.0001, timeout=.0001).pop()], [sys.getsizeof(L), sys.getsizeof(results)]
def memory_used(func, *args, **kwargs): """Compute memory usage when executing func.""" def func_3_times(*args, **kwargs): for _ in range(3): func(*args, **kwargs) gc.collect() mem_use = memory_usage((func_3_times, args, kwargs), interval=0.001) return max(mem_use) - min(mem_use)
def memory_used(func, *args, **kwargs): """Compute memory usage of func.""" if memory_usage is None: return np.NaN gc.collect() mem_use = memory_usage((func, args, kwargs), interval=.001) return max(mem_use) - min(mem_use)
def memory(message=None,className='',memSaved=None): global memLog global memLast global memList global memMax if memLog: gc.collect() from memory_profiler import memory_usage memList = memory_usage(-1) if len(memList) < 1: memList = memory_usage(-1,timeout=10) assert(len(memList) > 0) mem = memList[-1] if mem > memMax: memMax = mem if memSaved != None: memInc = mem - memSaved else: memInc = mem-memLast memList.append((message,memInc)) memLast = mem caller = inspect.stack()[1][3] line = inspect.stack()[1][2] filename = inspect.stack()[1][1] if className: className += '.' if memHardLimit: if mem > memHardLimit: import sys from mpi4py import MPI if className == None: className = "UnknownClass" if caller == None: caller="UnknownCaller" if line == None: line = "Unknown Line" if message == None: message = '' logEvent("PROTEUS ERROR: MEMORY HARDLIMIT REACHED, EXIT From "+filename.split("/")[-1]+", "+className+caller+", line "+`line`+": "+message+", %f MB in routine, %f MB in program, %f MB is hard limit" % (memInc,mem,memHardLimit)) MPI.COMM_WORLD.Abort(1) sys.exit("MPI.COMM_WORLD.Abort(1); exit(1)") if message: return "In "+filename.split("/")[-1]+", "+className+caller+", line "+`line`+": "+message+", %f MB in routine, %f MB in program" % (memInc,mem)
def test(funcs, n=10): # run a few times to get any ancillary objects created for i in range(3): for f in funcs: f() gc.collect() memory_usage(-1, interval=0) a = np.zeros(n) (a.max() - a.min())/n for f in funcs: a = np.zeros(n) for i in range(n): f() gc.collect() r = memory_usage(-1, interval=0) a[i] = r[0] yield (a.max() - a.min())/n
def profile_expose_method(profiled_method_wrapper, accept, args, func, kw, exclude_from_memory_profiling): """ Targeted to profile a specific method that wraps HTTP request processing endpoints into database context. :param profiled_method_wrapper: method wrapped around profiled call to be passed in to memory profiler :param accept: param specific to profiled call :param args: args of a function that is being wrapped by a profiled method :param func: function that is being wrapped by a profiled method :param kw: kwargs of a function that is being wrapped by a profiled method :return: output of a profiled method without modification """ if not exclude_from_memory_profiling and get_memory_profile_logging_on() and \ check_memory_profile_package_wide_disable(func): controller_class = args[0].__class__.__name__ if args and len(args) > 0 else '' end_point_name_parts = [s for s in [func.__module__, controller_class, func.__name__] if s != ''] end_point_name = ".".join(end_point_name_parts) is_pympler_on = _is_pympler_profiling_value_on(end_point_name) profile_output = {'output': {}} if is_pympler_on: all_objects = muppy.get_objects() all_objects_summary_before = summary.summarize(all_objects) memory_profile = memory_usage((_profile_me, (profile_output, profiled_method_wrapper, func, accept, args, kw), {}), interval=0.1) output = profile_output['output'] if is_pympler_on: all_objects_summary_after = summary.summarize(all_objects) diff = summary.get_diff(all_objects_summary_before, all_objects_summary_after) diff_less = summary.format_(diff) diff_out = '' for s in diff_less: diff_out += s+'\n' thread_log.info("================ PYMPLER OUTPUT <{}> ==============\n{}".format(end_point_name, diff_out)) try: message = json.dumps({'log_type': 'memory_profile', 'proc_id': os.getpid(), 'name': func.__name__, 'module': func.__module__, 'mem_profile': memory_profile, 'min': min(memory_profile), 'max': max(memory_profile), 'diff': max(memory_profile) - min(memory_profile), 'leaked': memory_profile[-1] - memory_profile[0], 'args': [arg for arg in args[1:]], # exclude self 'kwargs': kw}) memory_log.info(message, extra={'controller_module': func.__module__, 'controller_class': controller_class, 'endpoint': func.__name__}) except Exception as e: thread_log.exception('Logger failed: {}'.format(e)) else: output = profiled_method_wrapper(accept, args, func, kw) return output
def memorySummary(): global memLog global memList global memMax if memLog: gc.collect() from memory_profile import memory_usage mem = memory_usage(-1)[0] if mem > 0: for pair in memList: logEvent(`pair[0]`+" %"+`100.0*pair[1]/memMax`)
def _get_memory_base(gallery_conf): """Get the base amount of memory used by running a Python process.""" if not gallery_conf['show_memory']: memory_base = 0 else: # There might be a cleaner way to do this at some point from memory_profiler import memory_usage proc = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(1.0)']) memory_base = max(memory_usage(proc, interval=1e-3, timeout=0.1)) return memory_base
def test_leak(self): sound, fs = self._load_wave('leak-test.wav') frame_ms = 0.010 frame_len = int(round(fs * frame_ms)) n = int(len(sound) / (2 * frame_len)) nrepeats = 1000 vad = webrtcvad.Vad(3) used_memory_before = memory_usage(-1)[0] for counter in range(nrepeats): find_voice = False for frame_ind in range(n): slice_start = (frame_ind * 2 * frame_len) slice_end = ((frame_ind + 1) * 2 * frame_len) if vad.is_speech(sound[slice_start:slice_end], fs): find_voice = True self.assertTrue(find_voice) used_memory_after = memory_usage(-1)[0] self.assertGreaterEqual( used_memory_before / 5.0, used_memory_after - used_memory_before)
def profile_tests(): test_functions = [fn for fn in inspect.getmembers(vector_test, inspect.isfunction) if fn[0].startswith('test_')] for name, fn in test_functions: # There are a couple of tests that are not run for the C implementation, skip those fn_args = inspect.getargspec(fn)[0] if 'pvector' in fn_args: print('Executing %s' % name) result = memory_profiler.memory_usage((run_function, (fn,), {}), interval=.1) assert not detect_memory_leak(result), (name, result)
def run(self): jobs = Job.objects.due() prof_string = "" if not self.do_profile else "[%8.2f MB] " % memory_profiler.memory_usage()[0] if jobs: logging.info("%sRunning %d due jobs... (%s)" % (prof_string, jobs.count(), ", ".join(['"%s"' % job.name for job in jobs]))) call_command('cron') else: logging.debug("%sNo jobs due to run." % prof_string) if self.do_gc: gc.collect()
def with_memory(*args, **kwargs): log = logging.getLogger(__name__) mem_usage, result = memory_usage( (func, args, kwargs), interval=0.1, timeout=1, max_usage=True, retval=True, include_children=True, ) log.info( "Running %r consumed %2.2fMiB memory at peak time", _func_full_name(func), mem_usage[0], ) return result
def Combiner(L): global lock lock.acquire() data = L[0] sizeOut = 0 for line in data: sizeOut += sys.getsizeof(data[line]) try: L[1][line].append(data[line]) except KeyError: L[1][line] = [data[line]] lock.release() return [ multiprocessing.current_process().name, memory_usage(-1, interval=.0001, timeout=.0001).pop() ], [sys.getsizeof(L), sizeOut]
def test_tokenize_big_file_lowest_memory(self): """ Tests that tokenize_big_file works efficiently than given memory reference """ expected = 224.9296875 * 1.1 actual_memory = memory_usage((tokenize_big_file, ('lab_2/data.txt', )), interval=2) actual = sum(actual_memory) / len(actual_memory) print( f'Actual tokenize_big_file function memory consuming is: {actual}') print( f'Reference tokenize_big_file function memory consuming is: {expected}' ) self.assertGreater(expected, actual)
def run_QUAD4Ms_series_mem(dq4ms, dwrks, douts, fq4rs, fdats, fouts, fbugs, nthreads=None): ''' Wrapper for run_QUAD4Ms_series that also tracks memory usage ''' args = (dq4ms, dwrks, douts, fq4rs, fdats, fouts, fbugs) mem = memory_usage(proc=(runQ4Ms_series, args), interval=0.5, timeout=2, include_children=True) return np.max(mem)
def inner(*args, **kwargs): mem_usage, ret = memory_usage( (fn, args, kwargs), max_usage=False, retval=True, timestamps=True, # TODO: check if this will get used when `proc` is a tuple. multiprocess=True, interval=INTERVAL, ) for line in mem_usage: fh.write(format_memline(line)) fh.write(format_funcline(fn.__qualname__, mem_usage)) return ret
def test_memory_usage_should_not_spike_on_app(n_runs, tolerance): """Call the app n times and ensure that mem usage is the same""" kwargs = { "n_iters": 1, "extinction": 0.125, "survival": 0.375, "size": 180, } avg_mem_usage_per_run = [ mean(memory_usage((make_sprite, [], kwargs), interval=0.2, timeout=1)) for _ in range(n_runs) ] first_run = avg_mem_usage_per_run[0] last_run = avg_mem_usage_per_run[-1] print(f"0: {first_run}, n:{last_run}") assert (last_run - first_run) < tolerance
def measure_dfs_recurrent(): dfs = DFS(Graph(0)) avg_memory_consumption = [] for i in range(START_COUNTING_FROM_NODE, NUM_OF_NODES): print(f"Measuring DFS Recurrent for {i} nodes.") graph = Graph(i) dfs.graph = graph starting_node = next(iter(graph.graph)) mem_usage = memory_usage((dfs.depth_first_search, (), { 'start_node': starting_node })) avg_memory_consumption.append(mem_usage[-1]) return avg_memory_consumption
def measure_bfs(): bfs = BFS(Graph(0)) avg_memory_consumption = [] for i in range(START_COUNTING_FROM_NODE, NUM_OF_NODES): print(f"Measuring BFS for {i} nodes.") graph = Graph(i) bfs.graph = graph starting_node = next(iter(graph.graph)) mem_usage = memory_usage((bfs.breadth_first_search, (), { 'start_node': starting_node })) avg_memory_consumption.append(mem_usage[-1]) return avg_memory_consumption
def test_memory_usage_ok(): import memory_profiler dataset = DataSet(width=80, height=80, max_steps=100000, phi_length=4) last = time.time() for i in xrange(1000000000): if (i % 100000) == 0: print i dataset.add_sample(np.random.random((80, 80)), 1, 1, False) if i > 200000: states, actions, rewards, next_states, terminals = \ dataset.random_batch(32) if (i % 10007) == 0: print time.time() - last mem_usage = memory_profiler.memory_usage(-1) print len(dataset), mem_usage last = time.time()
def main(): import shutil import tempfile import warnings from pandas import Series from vbench.api import BenchmarkRunner from suite import (REPO_PATH, BUILD, DB_PATH, PREPARE, dependencies, benchmarks) from memory_profiler import memory_usage warnings.filterwarnings('ignore', category=FutureWarning) try: TMP_DIR = tempfile.mkdtemp() runner = BenchmarkRunner( benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH, TMP_DIR, PREPARE, always_clean=True, # run_option='eod', start_date=START_DATE, module_dependencies=dependencies) results = {} for b in runner.benchmarks: k = b.name try: vs = memory_usage((b.run, )) v = max(vs) # print(k, v) results[k] = v except Exception as e: print("Exception caught in %s\n" % k) print(str(e)) s = Series(results) s.sort() print(s) finally: shutil.rmtree(TMP_DIR)
def test_memory_large(self): obj, _ = self.fixture_1() obj["local_fp"] = generateRandomFixture(count=10**6) obj["output"] = TEMP_FOLDER_PATH + "output_large" # print(obj) self.correctness_test(lambda: (obj, None)) # Performing correctness_test m = memory_usage(lambda: self.correctness_test(lambda: (obj, None))) # get max memory usage after subtracting initial memory usage max_memory_usage = max([a - m[0] for a in m]) # get input file size file_size = os.path.getsize(obj["local_fp"]) / (1024 * 1024) # in MBs # Asserts that memory usage is atleast 10 times lesser as compared to file size self.assertLessEqual(max_memory_usage * 10, file_size)
def inner(*args, **kwargs): fn_kwargs_str = ', '.join(f'{k}={v}' for k, v in kwargs.items()) print(f'\n{fn.__name__}({fn_kwargs_str})') # Measure time t = time.perf_counter() retval = fn(*args, **kwargs) elapsed = time.perf_counter() - t print(f'Time {elapsed:0.4}') # Measure memory mem, retval = memory_usage((fn, args, kwargs), retval=True, timeout=200, interval=1e-7) print(f'Memory {max(mem) - min(mem)}') return retval
def profile(nrange, func): res = pd.DataFrame(index=nrange, columns=["Time [s]", "Memory Usage"]) for N in res.index: start = time() func(N) end = time() duration = end - start memory = memory_usage((func, (N,))) res.loc[N] = duration, max(memory) return res
def test_memory_use(): """Naive test that assumes memory use will never be more than 120 % of that for first 50 rows""" folder = os.path.split(__file__)[0] src = os.path.join(folder, "files", "very_large.xlsx") wb = openpyxl.load_workbook(src, use_iterators=True) ws = wb.get_active_sheet() initial_use = None for n, line in enumerate(ws.iter_rows()): if n % 50 == 0: use = memory_usage(proc=-1, interval=1)[0] if initial_use is None: initial_use = use assert use / initial_use < 1.2 print n, use
async def _status(self, ctx: commands.Context): """Information about the bot's status.""" uptime = time.time() - self.bot.uptime hours = uptime / 3600 minutes = (uptime / 60) % 60 seconds = uptime % 60 users = 0 channel = 0 try: commands_chart = sorted(self.bot.counter.items(), key=lambda t: t[1], reverse=False) top_command = commands_chart.pop() command_info = f'{sum(self.bot.counter.values())} (Top Command: {top_command[0]} [x{top_command[1]}])' except IndexError: command_info = str(sum(self.bot.counter.values())) bot_member = ctx.message.guild.get_member(self.bot.user.id) for guild in self.bot.guilds: users += len(guild.members) channel += len(guild.channels) embed = discord.Embed(colour=bot_member.top_role.colour) embed.add_field(name='Bot Creator', value=self.bot.owner.name, inline=False) embed.add_field( name='Uptime', value='{0:.0f} Hours, {1:.0f} Minutes, and {2:.0f} Seconds'.format( hours, minutes, seconds), inline=False) embed.add_field(name='Total Users', value=users) embed.add_field(name='Total Channels', value=channel) embed.add_field(name='Total Servers', value=str(len(self.bot.guilds))) embed.add_field(name='Command Usage', value=command_info) embed.add_field(name='Bot Version', value=self.bot.version) embed.add_field(name='Discord.py Version', value=discord.__version__) embed.add_field(name='Python Version', value=platform.python_version()) embed.add_field(name='Memory Usage', value='{} MB'.format(round(memory_usage(-1)[0], 3))) embed.add_field(name='Operating System', value='{} {} {}'.format(platform.system(), platform.release(), platform.version()), inline=False) await ctx.send(embed=embed)
def _memory_usage(func, gallery_conf): """Get memory usage of a function call.""" if gallery_conf['show_memory']: from memory_profiler import memory_usage assert callable(func) mem, out = memory_usage(func, max_usage=True, retval=True, multiprocess=True) try: mem = mem[0] # old MP always returned a list except TypeError: # 'float' object is not subscriptable pass else: out = func() mem = 0 return out, mem
def performance_testing(data: Sequence, tests_count: int): results_times = [] results_memories = [] occurences = [] for batch in data: times_of_batch = [] memories_of_batch = [] for _ in range(tests_count): performance_memory, vals = memory_usage( (aho_corasick, (batch[0], batch[1])), retval=True) occurrences, performance_time = vals times_of_batch.append(performance_time) memories_of_batch.append( max(performance_memory) - min(performance_memory)) results_times.append(times_of_batch) results_memories.append(memories_of_batch) return [results_times, results_memories, occurrences]
def __init__(self): # keep a global accounting for the last known memory usage # which is the reference point for the memory delta calculation self.previous_call_memory_usage = memory_profiler.memory_usage()[0] self.t1 = time.time() # will be set to current time later self.keep_watching = True self.peak_memory_usage = -1 self.peaked_memory_usage = -1 self.memory_delta = 0 self.time_delta = 0 self.watching_memory = True self.ip = get_ipython() self.input_cells = self.ip.user_ns['In'] self._measurements = namedtuple( 'Measurements', ['memory_delta', 'time_delta', 'memory_peak', 'memory_usage'], verbose=False)
async def mProfileTask(self): try: from memory_profiler import memory_usage assert self.app.cmdArgs.memprofiling is True except (ImportError, Exception): pass while not self.should_stop: await asyncio.sleep(10) lt = int(self.app.loop.time()) usage = memory_usage(-1, interval=.2, timeout=1) if usage: log.debug( f'Memory Usage (LT: {lt}): {usage[0]}' )
def __call__(self, trial, train_loader, test_loader, epoch: int, device, optimizer, scheduler) -> Tuple[float, float]: if self.stream is None: self.stream = Profiler.filename.open('w+') print(f"AdaS: Profiler: Writing csv to {Profiler.filename}") else: if self.trial != trial: self.stream.close() self.stream = Profiler.filename.open('w+') print(f"AdaS: Profiler: Writing csv to {Profiler.filename}") self.trial = trial self.gpu.update() self.pr.enable() result = memory_usage(proc=(self.function, (trial, train_loader, test_loader, epoch, device, optimizer, scheduler)), max_usage=True, retval=True) self.pr.disable() s = io.StringIO() sortby = SortKey.CUMULATIVE ps = pstats.Stats(self.pr, stream=s).sort_stats(sortby) ps.print_stats( "epoch_iteration|step|trial_iteration|test_main|evaluate") stats_list = pstats_to_dict(s.getvalue()) header = 'epoch,epoch_gpu_mem_used,epoch_gpu_temp,epoch_ram_used' content = f'{epoch},{self.gpu.mem_used - Profiler.base_mem_used},{self.gpu.temperature},{result[0]}' for stat in stats_list: header += f",{stat['name']}_n_calls" header += f",{stat['name']}_tot_time" header += f",{stat['name']}_per_call1" header += f",{stat['name']}_cum_time" header += f",{stat['name']}_per_call2" content += f",{stat['n_calls']}" content += f",{stat['tot_time']}" content += f",{stat['per_call1']}" content += f",{stat['cum_time']}" content += f",{stat['per_call2']}" header += '\n' content += '\n' if not self.header_written: self.stream.write(header) self.header_written = True self.stream.write(content) return result[1]
def watch_memory(): import time # bring in the global memory usage value from the previous iteration global previous_call_memory_usage, peak_memory_usage, keep_watching, perf_proc nbr_commands = len(In) new_memory_usage = memory_profiler.memory_usage()[0] memory_delta = new_memory_usage - previous_call_memory_usage keep_watching = False peaked_memory_usage = max(0, peak_memory_usage - new_memory_usage) # calculate time delta using global t1 (from the pre-run event) and current # time time_delta_secs = time.time() - t1 perf_values = [] if perf_proc: # pause if necessary to attempt to make sure we get a sample from perf... # as the 100ms min sample time and flushing oddness means I don't get a # sample very quickly for short-running tasks MIN_TIME_TO_GET_PERF_SAMPLE = 0.2 if time_delta_secs < MIN_TIME_TO_GET_PERF_SAMPLE: print("PAUSING to get perf sample for {}s".format( MIN_TIME_TO_GET_PERF_SAMPLE)) time.sleep(MIN_TIME_TO_GET_PERF_SAMPLE ) # pause until at least 0.1s has passed # if we have a valid perf running then capture that information perf_values = perf_process.finish_perf(perf_proc) cmd = In[nbr_commands - 1] # convert the results into a pretty string #output_template = "'{cmd}' used {memory_delta:0.4f} MiB RAM in {time_delta:0.2f}s, peaked {peaked_memory_usage:0.2f} MiB above current, total RAM usage {memory_usage:0.2f} MiB" output_template = "Used {memory_delta:0.4f} MiB RAM in {time_delta:0.2f}s, peaked {peaked_memory_usage:0.2f} MiB above current, total RAM usage {memory_usage:0.2f} MiB" output = output_template.format(time_delta=time_delta_secs, cmd=cmd, memory_delta=memory_delta, peaked_memory_usage=peaked_memory_usage, memory_usage=new_memory_usage) print(str(output)) if perf_values: perf_average = int(sum(perf_values) / float(time_delta_secs)) #print("perf value for {} averages to {:,}/second, raw samples:".format(perf_process.EVENT_TYPE, perf_average), perf_values) print("perf value for {} averages to {:,}/second".format( perf_process.EVENT_TYPE, perf_average)) else: print( "perf - no results to report, possibly the collection time was too short?" ) previous_call_memory_usage = new_memory_usage
def run_benchmark(size, density, radius, algorithm, type, seed): neurons = [] # 47% of neurons in size neurons.extend(big_sample(msn_d1, 0.475 * density * size**3)) # print('msn_d1 neurons added', len(neurons), 'total') # 47% of neurons in size neurons.extend(big_sample(msn_d2, 0.475 * density * size**3)) # print('msn_d2 neurons added', len(neurons), 'total') # 1% of neurons in size neurons.extend(big_sample(fs_pv, 0.01 * density * size**3)) # print('fs_pv neurons added', len(neurons), 'total') # rotate and translate randomly for n in neurons: axis = random_axis(n) theta = random() * 2 * pi transform = rotation_matrix(axis, theta) for nn in n: nn.transform(transform) # print(len(neurons), 'neurons rotated') for n in neurons: x, y, z = [1000 * (random() * size - size / 2) for _ in range(3)] for nn in n: nn.translate(x, y, z) # print(len(neurons), 'neurons translated') for n in neurons: for nn in n: nn.name = uuid4() # print(len(neurons), 'neurons renamed') for i in range(10): print(algorithm.__name__, size, density, radius, sep=',', end=',') tbefore = time() ramsamples = memory_usage((algorithm, (neurons, radius))) print(time() - tbefore, min(ramsamples), max(ramsamples), type, seed, sep=',')
def track_repeated_subset_memratio(self, n_obs, n_var, attr_set, subset_dim, index_kind): nviews = 5 base_size = get_anndata_memsize(self.adata) mem_recording = memory_usage( ( sedate(take_repeated_view, 0.005), (self.adata, ), { "dim": subset_dim, "index_kind": index_kind, "nviews": nviews, "ratio": 0.9, }, ), interval=0.001, ) return (np.max(mem_recording) - np.min(mem_recording)) / base_size
def StartMeMonitoring(pid, datasetName): filename = datasetName + '-memory.txt' print("Memory Logger Start\n") print("Result File: " + filename) """ if not os.path.exists(os.path.dirname(filename)): try: os.makedirs(os.path.dirname(filename)) except OSError as exc: # Guard against race condition if exc.errno != errno.EEXIST: raise """ f = open(filename, "a") mem_usage = memory_usage(int(pid), interval=1, timeout=3600000, retval=False, stream=f)
def profile_memory(): """Profile the memory usage of the Python program with memory_profiler""" from memory_profiler import profile from memory_profiler import memory_usage set_logger_debug() rpa_eb = RPA_EB(MODEL_PATH, SHAPE, BUF_SIZE) # Add decorators for potentially memory eating methods rpa_eb._run_test = profile(rpa_eb._run_test) rpa_eb._proc_frame = profile(rpa_eb._proc_frame) rpa_eb._merge_bboxes = profile(rpa_eb._merge_bboxes) def test_single_proc(): rpa_eb.run(test=True, multi_process=False, frame_num=1) test_single_proc() print("Total memory usage: {} MiB".format(memory_usage()[0]))
def test_main(): min_kb, max_kb = get_bounds() # This should be the **only** test function here. gc.disable() intersect_all() kb_used_after = memory_profiler.memory_usage(max_usage=True) if min_kb <= kb_used_after <= max_kb: status = 0 msg = SUCCESS_TEMPLATE.format(kb_used_after) print(msg) else: status = 1 msg = ERR_TEMPLATE.format(kb_used_after, min_kb, max_kb) print(msg, file=sys.stderr) gc.enable() sys.exit(status)
def Map(L): # print "Map:", multiprocessing.current_process().name, "\t", #print len(L) # save to global and process # workerLoad.append(memory_usage(-1, interval=.2, timeout=.2).pop()) # workerLoad = measureLoad() results = {} # key value storage for line in L: key = str(line[0] + ":" + line[1] + ":" + line[2]) try: results[key] += 1 except KeyError: results[key] = 1 print "Map: ", [sys.getsizeof(L), sys.getsizeof(results)] return results, [ multiprocessing.current_process().name, memory_usage(-1, interval=.0001, timeout=.0001).pop() ], [sys.getsizeof(L), sys.getsizeof(results)]
def profile_memory(fn): ''' Given the function to profile, profiles the memory usage for the function. Args: fn (python func): Function to profile Returns: Nothing (saves the memory profile of the function recorded at fixed intervals) ''' logging.info('PROFILING MEMORY ...') mem_usage = memory_usage(fn, interval=.01) logging.info('PROFILED MEMORY.') logging.info('max memory={:3f}MB.'.format(max(mem_usage))) pickle.dump(mem_usage, open('memory_example.out', 'wb'))
def inner(*args, **kwargs): fn_kwargs_str = ", ".join(f"{k}={v}" for k, v in kwargs.items()) print(f"\n{fn.__name__}({fn_kwargs_str})") # Measure time t = time.perf_counter() return_value = fn(*args, **kwargs) elapsed = time.perf_counter() - t print(f"Time spent: {elapsed:0.4}") # Measure memory mem, return_value = memory_usage((fn, args, kwargs), retval=True, timeout=200, interval=1e-7) print(f"Memory used: {max(mem) - min(mem)}") return return_value
def inner(*args, **kwargs): #fn_kwargs_str = ', '.join(f'{k}={v}' for k, v in kwargs.items()) # if you want to print the function name and kwargs - print(f'\n{fn.__name__}({fn_kwargs_str})') # Measure time and memory # set time start t = time.perf_counter() # execute function and capture memory usage mem, retval = memory_usage((fn, args, kwargs), retval=True, interval=1e-7) #timeout=200 elapsed = time.perf_counter() - t # print results of time and memory print(f'Time {elapsed:0.4}') print(f'Memory {max(mem) - min(mem)}') # return function output return retval