def checkpoint_memory():
'''This test is meant to be run manually, since it depends on
memory_profiler and its behavior may vary.'''
try:
from memory_profiler import memory_usage
except ImportError:
return
def f(a):
for _ in range(10):
a = np.sin(a**2 + 1)
return a
checkpointed_f = checkpoint(f)
def testfun(f, x):
for _ in range(5):
x = f(x)
return np.sum(x)
gradfun = grad(testfun, 1)
A = npr.RandomState(0).randn(100000)
max_usage = max(memory_usage((gradfun, (f, A))))
max_checkpointed_usage = max(memory_usage((gradfun, (checkpointed_f, A))))
assert max_checkpointed_usage < max_usage / 2.
def assertNotIncreasingMemory(self,
f,
num_iters=100000,
increase_threshold_absolute_mb=10):
"""Assert memory usage doesn't increase beyond given threshold for f."""
with context.eager_mode():
# Warm up.
f()
# Wait for background threads to start up and take over memory.
# FIXME: The nature of this test leaves few other options. Maybe there
# is a better way to do this.
time.sleep(4)
initial = memory_profiler.memory_usage(-1)[0]
for _ in six.moves.range(num_iters):
f()
increase = memory_profiler.memory_usage(-1)[0] - initial
assert increase < increase_threshold_absolute_mb, (
"Increase is too high. Initial memory usage: %f MB. Increase: %f MB. "
"Maximum allowed increase: %f") % (initial, increase,
increase_threshold_absolute_mb)
def trace_calls_and_returns(frame, event, arg):
global just_returned
co = frame.f_code
func_name = co.co_name
if just_returned:
just_returned = False
print 'just_returned'
print 'mem before gc ' + str(memory_usage(-1))
# gc.collect()
print 'mem after gc ' + str(memory_usage(-1))
if func_name == 'memory_usage':
return
if func_name == 'write':
# Ignore write() calls from print statements
return
line_no = frame.f_lineno
filename = co.co_filename
if event == 'call':
print 'Call to %s on line %s of %s' % (func_name, line_no, filename)
return trace_calls_and_returns
elif event == 'return':
print 'return of %s => %s' % (func_name, arg)
just_returned = True
print 'mem before gc ' + str(memory_usage(-1))
# gc.collect()
print 'mem after gc ' + str(memory_usage(-1))
elif event == 'line':
print 'a line event ' + str(line_no)
return
def memory_profile(function, *args, **kwargs):
gc.collect()
baseline = memory_profiler.memory_usage()[0]
max_usage = memory_profiler.memory_usage(
(function, args, kwargs),
max_usage=True,
)
return max_usage[0] - baseline
def run_tests_if_main(measure_mem=False):
"""Run tests in a given file if it is run as a script."""
local_vars = inspect.currentframe().f_back.f_locals
if not local_vars.get('__name__', '') == '__main__':
return
# we are in a "__main__"
try:
import faulthandler
faulthandler.enable()
except Exception:
pass
with warnings.catch_warnings(record=True): # memory_usage internal dep.
mem = int(round(max(memory_usage(-1)))) if measure_mem else -1
if mem >= 0:
print('Memory consumption after import: %s' % mem)
t0 = time.time()
peak_mem, peak_name = mem, 'import'
max_elapsed, elapsed_name = 0, 'N/A'
count = 0
for name in sorted(list(local_vars.keys()), key=lambda x: x.lower()):
val = local_vars[name]
if name.startswith('_'):
continue
elif callable(val) and name.startswith('test'):
count += 1
doc = val.__doc__.strip() if val.__doc__ else name
sys.stdout.write('%s ... ' % doc)
sys.stdout.flush()
try:
t1 = time.time()
if measure_mem:
with warnings.catch_warnings(record=True): # dep warn
mem = int(round(max(memory_usage((val, (), {})))))
else:
val()
mem = -1
if mem >= peak_mem:
peak_mem, peak_name = mem, name
mem = (', mem: %s MB' % mem) if mem >= 0 else ''
elapsed = int(round(time.time() - t1))
if elapsed >= max_elapsed:
max_elapsed, elapsed_name = elapsed, name
sys.stdout.write('time: %0.3f sec%s\n' % (elapsed, mem))
sys.stdout.flush()
except Exception as err:
if 'skiptest' in err.__class__.__name__.lower():
sys.stdout.write('SKIP (%s)\n' % str(err))
sys.stdout.flush()
else:
raise
elapsed = int(round(time.time() - t0))
sys.stdout.write('Total: %s tests\n• %0.3f sec (%0.3f sec for %s)\n• '
'Peak memory %s MB (%s)\n'
% (count, elapsed, max_elapsed, elapsed_name, peak_mem,
peak_name))
def test_memory_leak():
"""https://github.com/emcconville/wand/pull/127"""
minimum = 1.0
with Color('NONE') as nil_color:
minimum = ctypes.sizeof(nil_color.raw)
consumes = memory_usage((color_memory_leak, (), {}))
assert consumes[-1] - consumes[0] <= minimum
def custom_minimize(function, algorithm, bounds = None, guess = None):
def iter_minimize(): # lightweight version of iter_minimize for a single optimization method
start = timeit.default_timer()
result = 0
# some minimization techniques do not require an initial guess
if guess is not None:
result = algorithm(function, guess)
else:
result = algorithm(function, bounds)
iterations = -1
if 'nit' in result.keys():
iterations = result.get('nit')
stop = timeit.default_timer()
return iterations, start, stop, result
#tracks amount of memory used by current process (-1) every interval (.2 seconds)
current_memory = memory_usage(-1, interval=.2)
most_mem = np.append(most_mem, max(current_memory))
num_iters, start, stop, result = iter_minimize()
exec_time = stop-start
print '{0} took {1} seconds. The result, {2} was found at ({3})'.format(algorithm.__name__,exec_time,result.fun,result.x)
print '{0} used {1} megabytes and took {2} iterations'.format(algorithm.__name__,most_mem,num_iters)
print
def during_execution_memory_sampler():
import time
import memory_profiler
global keep_watching, peak_memory_usage
peak_memory_usage = -1
keep_watching = True
n = 0
WAIT_BETWEEN_SAMPLES_SECS = 0.001
MAX_ITERATIONS = 60.0 / WAIT_BETWEEN_SAMPLES_SECS
while True:
mem_usage = memory_profiler.memory_usage()[0]
peak_memory_usage = max(mem_usage, peak_memory_usage)
time.sleep(WAIT_BETWEEN_SAMPLES_SECS)
if not keep_watching or n > MAX_ITERATIONS:
# exit if we've been told our command has finished or if it has run
# for more than a sane amount of time (e.g. maybe something crashed
# and we don't want this to carry on running)
if n > MAX_ITERATIONS:
print(
"{} SOMETHING WEIRD HAPPENED AND THIS RAN FOR TOO LONG, THIS THREAD IS KILLING ITSELF".format(
__file__
)
)
break
n += 1
def test(prefix, item_ct, loop_ct):
h5 = h5py.File(sys.argv[1], "a")
cc = h5.id.get_mdc_config()
print(cc.max_size)
print(h5.id.get_access_plist().get_cache())
cc.min_size = 1024 * 1024
cc.max_size = 1024 * 1024
h5.id.set_mdc_config(cc)
# h5.create_group(b'\xff') # force new compact-or-indexed group with high byte
# del h5[b'\xff']
print("start %s %.3f" % (prefix, time.time() - start))
path_i = 0
for i in range(loop_ct):
data = np.zeros(item_ct, dtype=np.float32)
i_str = "%05d" % i
# path = ''
# path = '/'.join((prefix, i_str[0], i_str[1], i_str[2], i_str[3]))
path = str(path_i)
# print(path + '/' + str(i))
ds = h5.create_dataset(path + "/" + str(i), data=data)
# ds = h5.create_dataset(path + '/' + str(i), (item_ct,), dtype=np.float32)
# ds[:] = data
# h5.create_group(path + '/' + i_str)
if i != 0 and i % close_interval == 0:
path_i += 1
# print(i, '%.1f' % (time.time() - start), h5.id.get_mdc_size(), h5.id.get_mdc_hit_rate())
print(
i,
"%.1f" % (time.time() - start),
h5.id.get_mdc_size(),
h5.id.get_mdc_hit_rate(),
memory_profiler.memory_usage(-1),
)
print("end %s %.3f" % (prefix, time.time() - start))
h5.close()
def __init__(self, method, n=1, timeout=120.0):
from memory_profiler import memory_usage
import time
from util import avg
self.timeout = timeout
max_memory_list = []
avg_memory_list = []
time_list = []
for i in range(n):
runner = self._Runner(memory_usage, {'proc': method, 'interval': 0.01, 'retval': True}, timeout)
baseline = avg(memory_usage((lambda: None, ()))) # baseline memory usage measurement
start_time = time.time()
runner.run() # actual running of the method
end_time = time.time()
if runner.result is not None:
mem_use, self.result = runner.result
memory = map(lambda x: x - baseline, mem_use)
else: # there was a timeout
import sys
memory = [-sys.maxint]
self.result = None
max_memory_list.append(max(memory))
avg_memory_list.append(avg(memory))
time_list.append((end_time - start_time) * 1000)
self.max_memory = avg(max_memory_list)
self.avg_memory = avg(avg_memory_list)
self.time = avg(time_list)
def run(self):
jobs = Job.objects.due()
if self.do_profile:
try:
import memory_profiler
prof_string = "[%8.2f MB] " % memory_profiler.memory_usage()[0]
except ImportError:
prof_string = "No profiler found"
else:
prof_string = ""
if jobs:
logging.info(
"%sRunning %d due jobs... (%s)"
% (prof_string, jobs.count(), ", ".join(['"%s"' % job.name for job in jobs]))
)
for job in Job.objects.due():
job.run()
else:
logging.debug("%sNo jobs due to run." % prof_string)
if self.do_gc:
gc.collect()
def time_svd(svdfunc, N1, N2, f, rseed=0, bestof=3, args=None, matfunc=np.asarray, **kwargs):
if args is None:
args = ()
N1_N2_f = np.broadcast(N1, N2, f)
times = []
memory = []
for (N1, N2, f) in N1_N2_f:
M = sparse_matrix(N1, N2, f, matfunc, rseed)
t_best = np.inf
mem_best = np.inf
for i in range(bestof):
t0 = time()
if args:
_args = [M]
_args.extend(list(args))
else:
_args = (M,)
mem_usage = max(memory_usage((svdfunc, _args, kwargs)))
t1 = time()
t_best = min(t_best, t1 - t0)
mem_best = min(mem_best, mem_usage)
times.append(t_best)
memory.append(mem_best)
return np.array(times).reshape(N1_N2_f.shape), np.array(memory).reshape(N1_N2_f.shape)
def run_example(self, theInput, theOutput):
# Import program (decrapted in 3.4, no other way at the moment)
from importlib.machinery import SourceFileLoader
solution = SourceFileLoader("solution", self.programPath).load_module()
# Feed the input
with PatchStd(theInput) as std:
# Start time counter
startTime = time.time()
# Run the program
solution.main()
# Get end time
endTime = time.time() - startTime
# Get memory (include current tests ~14MB but more or less is that)
mem = memory_usage(max_usage=True)
# Check output
actual_output = std.getStdOut().getvalue()
self.assertEqual(actual_output, theOutput)
# Print time (not do before because output is not yet retrieved)
std.restore()
print("\tTime: %.3f sec" % endTime)
print("\tMemory: %.3f MB" % mem)
# Show errors if any
errors = std.getStdErr().getvalue()
if errors != '':
print("\t" + errors)
def watch_memory():
import time
# bring in the global memory usage value from the previous iteration
global previous_call_memory_usage, peak_memory_usage, keep_watching
nbr_commands = len(In)
new_memory_usage = memory_profiler.memory_usage()[0]
memory_delta = new_memory_usage - previous_call_memory_usage
keep_watching = False
peaked_memory_usage = max(0, peak_memory_usage - new_memory_usage)
# calculate time delta using global t1 (from the pre-run event) and current
# time
time_delta_secs = time.time() - t1
cmd = In[nbr_commands - 1]
# convert the results into a pretty string
output_template = "'{cmd}' used {memory_delta:0.4f} MiB RAM in {time_delta:0.2f}s, peaked {peaked_memory_usage:0.2f} MiB above current, total RAM usage {memory_usage:0.2f} MiB"
output = output_template.format(
time_delta=time_delta_secs,
cmd=cmd,
memory_delta=memory_delta,
peaked_memory_usage=peaked_memory_usage,
memory_usage=new_memory_usage,
)
print(str(output))
previous_call_memory_usage = new_memory_usage
def f(a, v):
print "running", os.getpid(), len(a)
print "in process", memory_profiler.memory_usage()
#a[0] = 'x'
print "did arr get changed in process?", a[:5]
v.value = len(a)
time.sleep(5)
def memorySummary():
global memLog
global memList
global memMax
if memLog:
gc.collect()
try:
from memory_profile import memory_usage
mem = memory_usage(-1)[0]
except:
raise
# except:
# try:
# mem = int(os.popen('ps -p %d -o vsz|tail -1' % os.getpid()).read())/1024.0
# except:
# try:
# #CLE doesn't have full ps functionality
# #try to find first occurence of pid, then get memory usage slot
# #mwf debug
# #import pdb
# #pdb.set_trace()
# plist = os.popen('ps ').read()
# pr = plist.split('%s' % os.getpid())
# #print 'pr = %s ' % pr
# #print 'pr[1] = %s ' % pr[1]
# mem = int(pr[1].split()[1])/1024.0
# except:
# logEvent("memory function doesn't work on this platform\n")
# mem = 0
if mem > 0:
for pair in memList:
logEvent(`pair[0]`+" %"+`100.0*pair[1]/memMax`)
def watch_memory():
# bring in the global memory usage value from the previous iteration
global previous_call_memory_usage, peak_memory_usage, keep_watching, \
watching_memory, input_cells
new_memory_usage = memory_profiler.memory_usage()[0]
memory_delta = new_memory_usage - previous_call_memory_usage
keep_watching = False
peaked_memory_usage = max(0, peak_memory_usage - new_memory_usage)
# calculate time delta using global t1 (from the pre-run event) and current
# time
time_delta_secs = time.time() - t1
num_commands = len(input_cells) - 1
cmd = "In [{}]".format(num_commands)
# convert the results into a pretty string
output_template = ("{cmd} used {memory_delta:0.4f} MiB RAM in "
"{time_delta:0.2f}s, peaked {peaked_memory_usage:0.2f} "
"MiB above current, total RAM usage "
"{memory_usage:0.2f} MiB")
output = output_template.format(time_delta=time_delta_secs,
cmd=cmd,
memory_delta=memory_delta,
peaked_memory_usage=peaked_memory_usage,
memory_usage=new_memory_usage)
if watching_memory:
print(str(output))
previous_call_memory_usage = new_memory_usage
def watch_memory(self):
if not self.watching_memory:
return
# calculate time delta using global t1 (from the pre-run
# event) and current time
self.time_delta = time.time() - self.t1
new_memory_usage = memory_profiler.memory_usage()[0]
self.memory_delta = new_memory_usage - self.previous_call_memory_usage
self.keep_watching = False
self.peaked_memory_usage = max(0, self.peak_memory_usage - new_memory_usage)
num_commands = len(self.input_cells) - 1
cmd = "In [{}]".format(num_commands)
# convert the results into a pretty string
output_template = ("{cmd} used {memory_delta:0.3f} MiB RAM in "
"{time_delta:0.3f}s, peaked {peaked_memory_usage:0.3f} "
"MiB above current, total RAM usage "
"{memory_usage:0.3f} MiB")
output = output_template.format(
time_delta=self.time_delta,
cmd=cmd,
memory_delta=self.memory_delta,
peaked_memory_usage=self.peaked_memory_usage,
memory_usage=new_memory_usage)
print(str(output))
self.previous_call_memory_usage = new_memory_usage
def leak():
data = [pa.array(np.concatenate([np.random.randn(100000)] * 1000))]
table = pa.Table.from_arrays(data, ['foo'])
while True:
print('calling to_pandas')
print('memory_usage: {0}'.format(memory_profiler.memory_usage()))
table.to_pandas()
gc.collect()
def Map(L):
results = {} # key value storage
for line in L:
key = str(line[0] +":" + line[1] +":" + line[2]);
try:
results[key] += 1
except KeyError:
results[key] = 1
return results, [multiprocessing.current_process().name, memory_usage(-1, interval=.0001, timeout=.0001).pop()], [sys.getsizeof(L), sys.getsizeof(results)]
def memory_used(func, *args, **kwargs):
"""Compute memory usage when executing func."""
def func_3_times(*args, **kwargs):
for _ in range(3):
func(*args, **kwargs)
gc.collect()
mem_use = memory_usage((func_3_times, args, kwargs), interval=0.001)
return max(mem_use) - min(mem_use)
def memory_used(func, *args, **kwargs):
"""Compute memory usage of func."""
if memory_usage is None:
return np.NaN
gc.collect()
mem_use = memory_usage((func, args, kwargs), interval=.001)
return max(mem_use) - min(mem_use)
def memory(message=None,className='',memSaved=None):
global memLog
global memLast
global memList
global memMax
if memLog:
gc.collect()
from memory_profiler import memory_usage
memList = memory_usage(-1)
if len(memList) < 1:
memList = memory_usage(-1,timeout=10)
assert(len(memList) > 0)
mem = memList[-1]
if mem > memMax:
memMax = mem
if memSaved != None:
memInc = mem - memSaved
else:
memInc = mem-memLast
memList.append((message,memInc))
memLast = mem
caller = inspect.stack()[1][3]
line = inspect.stack()[1][2]
filename = inspect.stack()[1][1]
if className:
className += '.'
if memHardLimit:
if mem > memHardLimit:
import sys
from mpi4py import MPI
if className == None:
className = "UnknownClass"
if caller == None:
caller="UnknownCaller"
if line == None:
line = "Unknown Line"
if message == None:
message = ''
logEvent("PROTEUS ERROR: MEMORY HARDLIMIT REACHED, EXIT From "+filename.split("/")[-1]+", "+className+caller+", line "+`line`+": "+message+", %f MB in routine, %f MB in program, %f MB is hard limit" % (memInc,mem,memHardLimit))
MPI.COMM_WORLD.Abort(1)
sys.exit("MPI.COMM_WORLD.Abort(1); exit(1)")
if message:
return "In "+filename.split("/")[-1]+", "+className+caller+", line "+`line`+": "+message+", %f MB in routine, %f MB in program" % (memInc,mem)
def test(funcs, n=10):
# run a few times to get any ancillary objects created
for i in range(3):
for f in funcs:
f()
gc.collect()
memory_usage(-1, interval=0)
a = np.zeros(n)
(a.max() - a.min())/n
for f in funcs:
a = np.zeros(n)
for i in range(n):
f()
gc.collect()
r = memory_usage(-1, interval=0)
a[i] = r[0]
yield (a.max() - a.min())/n
def profile_expose_method(profiled_method_wrapper, accept, args, func, kw, exclude_from_memory_profiling):
"""
Targeted to profile a specific method that wraps HTTP request processing endpoints into database context.
:param profiled_method_wrapper: method wrapped around profiled call to be passed in to memory profiler
:param accept: param specific to profiled call
:param args: args of a function that is being wrapped by a profiled method
:param func: function that is being wrapped by a profiled method
:param kw: kwargs of a function that is being wrapped by a profiled method
:return: output of a profiled method without modification
"""
if not exclude_from_memory_profiling and get_memory_profile_logging_on() and \
check_memory_profile_package_wide_disable(func):
controller_class = args[0].__class__.__name__ if args and len(args) > 0 else ''
end_point_name_parts = [s for s in [func.__module__, controller_class, func.__name__] if s != '']
end_point_name = ".".join(end_point_name_parts)
is_pympler_on = _is_pympler_profiling_value_on(end_point_name)
profile_output = {'output': {}}
if is_pympler_on:
all_objects = muppy.get_objects()
all_objects_summary_before = summary.summarize(all_objects)
memory_profile = memory_usage((_profile_me,
(profile_output, profiled_method_wrapper, func, accept, args, kw),
{}),
interval=0.1)
output = profile_output['output']
if is_pympler_on:
all_objects_summary_after = summary.summarize(all_objects)
diff = summary.get_diff(all_objects_summary_before, all_objects_summary_after)
diff_less = summary.format_(diff)
diff_out = ''
for s in diff_less:
diff_out += s+'\n'
thread_log.info("================ PYMPLER OUTPUT <{}> ==============\n{}".format(end_point_name, diff_out))
try:
message = json.dumps({'log_type': 'memory_profile',
'proc_id': os.getpid(),
'name': func.__name__,
'module': func.__module__,
'mem_profile': memory_profile,
'min': min(memory_profile),
'max': max(memory_profile),
'diff': max(memory_profile) - min(memory_profile),
'leaked': memory_profile[-1] - memory_profile[0],
'args': [arg for arg in args[1:]], # exclude self
'kwargs': kw})
memory_log.info(message,
extra={'controller_module': func.__module__,
'controller_class': controller_class,
'endpoint': func.__name__})
except Exception as e:
thread_log.exception('Logger failed: {}'.format(e))
else:
output = profiled_method_wrapper(accept, args, func, kw)
return output
def memorySummary():
global memLog
global memList
global memMax
if memLog:
gc.collect()
from memory_profile import memory_usage
mem = memory_usage(-1)[0]
if mem > 0:
for pair in memList:
logEvent(`pair[0]`+" %"+`100.0*pair[1]/memMax`)
def _get_memory_base(gallery_conf):
"""Get the base amount of memory used by running a Python process."""
if not gallery_conf['show_memory']:
memory_base = 0
else:
# There might be a cleaner way to do this at some point
from memory_profiler import memory_usage
proc = subprocess.Popen([sys.executable, '-c',
'import time; time.sleep(1.0)'])
memory_base = max(memory_usage(proc, interval=1e-3, timeout=0.1))
return memory_base
def test_leak(self):
sound, fs = self._load_wave('leak-test.wav')
frame_ms = 0.010
frame_len = int(round(fs * frame_ms))
n = int(len(sound) / (2 * frame_len))
nrepeats = 1000
vad = webrtcvad.Vad(3)
used_memory_before = memory_usage(-1)[0]
for counter in range(nrepeats):
find_voice = False
for frame_ind in range(n):
slice_start = (frame_ind * 2 * frame_len)
slice_end = ((frame_ind + 1) * 2 * frame_len)
if vad.is_speech(sound[slice_start:slice_end], fs):
find_voice = True
self.assertTrue(find_voice)
used_memory_after = memory_usage(-1)[0]
self.assertGreaterEqual(
used_memory_before / 5.0,
used_memory_after - used_memory_before)
def profile_tests():
test_functions = [fn for fn in inspect.getmembers(vector_test, inspect.isfunction)
if fn[0].startswith('test_')]
for name, fn in test_functions:
# There are a couple of tests that are not run for the C implementation, skip those
fn_args = inspect.getargspec(fn)[0]
if 'pvector' in fn_args:
print('Executing %s' % name)
result = memory_profiler.memory_usage((run_function, (fn,), {}), interval=.1)
assert not detect_memory_leak(result), (name, result)
def run(self):
jobs = Job.objects.due()
prof_string = "" if not self.do_profile else "[%8.2f MB] " % memory_profiler.memory_usage()[0]
if jobs:
logging.info("%sRunning %d due jobs... (%s)" % (prof_string, jobs.count(), ", ".join(['"%s"' % job.name for job in jobs])))
call_command('cron')
else:
logging.debug("%sNo jobs due to run." % prof_string)
if self.do_gc:
gc.collect()
def with_memory(*args, **kwargs):
log = logging.getLogger(__name__)
mem_usage, result = memory_usage(
(func, args, kwargs),
interval=0.1,
timeout=1,
max_usage=True,
retval=True,
include_children=True,
)
log.info(
"Running %r consumed %2.2fMiB memory at peak time",
_func_full_name(func),
mem_usage[0],
)
return result
def Combiner(L):
global lock
lock.acquire()
data = L[0]
sizeOut = 0
for line in data:
sizeOut += sys.getsizeof(data[line])
try:
L[1][line].append(data[line])
except KeyError:
L[1][line] = [data[line]]
lock.release()
return [
multiprocessing.current_process().name,
memory_usage(-1, interval=.0001, timeout=.0001).pop()
], [sys.getsizeof(L), sizeOut]
def test_tokenize_big_file_lowest_memory(self):
"""
Tests that tokenize_big_file
works efficiently than given memory reference
"""
expected = 224.9296875 * 1.1
actual_memory = memory_usage((tokenize_big_file, ('lab_2/data.txt', )),
interval=2)
actual = sum(actual_memory) / len(actual_memory)
print(
f'Actual tokenize_big_file function memory consuming is: {actual}')
print(
f'Reference tokenize_big_file function memory consuming is: {expected}'
)
self.assertGreater(expected, actual)
def run_QUAD4Ms_series_mem(dq4ms,
dwrks,
douts,
fq4rs,
fdats,
fouts,
fbugs,
nthreads=None):
''' Wrapper for run_QUAD4Ms_series that also tracks memory usage '''
args = (dq4ms, dwrks, douts, fq4rs, fdats, fouts, fbugs)
mem = memory_usage(proc=(runQ4Ms_series, args),
interval=0.5,
timeout=2,
include_children=True)
return np.max(mem)
def inner(*args, **kwargs):
mem_usage, ret = memory_usage(
(fn, args, kwargs),
max_usage=False,
retval=True,
timestamps=True,
# TODO: check if this will get used when `proc` is a tuple.
multiprocess=True,
interval=INTERVAL,
)
for line in mem_usage:
fh.write(format_memline(line))
fh.write(format_funcline(fn.__qualname__, mem_usage))
return ret
def test_memory_usage_should_not_spike_on_app(n_runs, tolerance):
"""Call the app n times and ensure that mem usage is the same"""
kwargs = {
"n_iters": 1,
"extinction": 0.125,
"survival": 0.375,
"size": 180,
}
avg_mem_usage_per_run = [
mean(memory_usage((make_sprite, [], kwargs), interval=0.2, timeout=1))
for _ in range(n_runs)
]
first_run = avg_mem_usage_per_run[0]
last_run = avg_mem_usage_per_run[-1]
print(f"0: {first_run}, n:{last_run}")
assert (last_run - first_run) < tolerance
def measure_dfs_recurrent():
dfs = DFS(Graph(0))
avg_memory_consumption = []
for i in range(START_COUNTING_FROM_NODE, NUM_OF_NODES):
print(f"Measuring DFS Recurrent for {i} nodes.")
graph = Graph(i)
dfs.graph = graph
starting_node = next(iter(graph.graph))
mem_usage = memory_usage((dfs.depth_first_search, (), {
'start_node': starting_node
}))
avg_memory_consumption.append(mem_usage[-1])
return avg_memory_consumption
def measure_bfs():
bfs = BFS(Graph(0))
avg_memory_consumption = []
for i in range(START_COUNTING_FROM_NODE, NUM_OF_NODES):
print(f"Measuring BFS for {i} nodes.")
graph = Graph(i)
bfs.graph = graph
starting_node = next(iter(graph.graph))
mem_usage = memory_usage((bfs.breadth_first_search, (), {
'start_node': starting_node
}))
avg_memory_consumption.append(mem_usage[-1])
return avg_memory_consumption
def test_memory_usage_ok():
import memory_profiler
dataset = DataSet(width=80, height=80, max_steps=100000, phi_length=4)
last = time.time()
for i in xrange(1000000000):
if (i % 100000) == 0:
print i
dataset.add_sample(np.random.random((80, 80)), 1, 1, False)
if i > 200000:
states, actions, rewards, next_states, terminals = \
dataset.random_batch(32)
if (i % 10007) == 0:
print time.time() - last
mem_usage = memory_profiler.memory_usage(-1)
print len(dataset), mem_usage
last = time.time()
def main():
import shutil
import tempfile
import warnings
from pandas import Series
from vbench.api import BenchmarkRunner
from suite import (REPO_PATH, BUILD, DB_PATH, PREPARE, dependencies,
benchmarks)
from memory_profiler import memory_usage
warnings.filterwarnings('ignore', category=FutureWarning)
try:
TMP_DIR = tempfile.mkdtemp()
runner = BenchmarkRunner(
benchmarks,
REPO_PATH,
REPO_PATH,
BUILD,
DB_PATH,
TMP_DIR,
PREPARE,
always_clean=True,
# run_option='eod', start_date=START_DATE,
module_dependencies=dependencies)
results = {}
for b in runner.benchmarks:
k = b.name
try:
vs = memory_usage((b.run, ))
v = max(vs)
# print(k, v)
results[k] = v
except Exception as e:
print("Exception caught in %s\n" % k)
print(str(e))
s = Series(results)
s.sort()
print(s)
finally:
shutil.rmtree(TMP_DIR)
def test_memory_large(self):
obj, _ = self.fixture_1()
obj["local_fp"] = generateRandomFixture(count=10**6)
obj["output"] = TEMP_FOLDER_PATH + "output_large"
# print(obj)
self.correctness_test(lambda: (obj, None))
# Performing correctness_test
m = memory_usage(lambda: self.correctness_test(lambda: (obj, None)))
# get max memory usage after subtracting initial memory usage
max_memory_usage = max([a - m[0] for a in m])
# get input file size
file_size = os.path.getsize(obj["local_fp"]) / (1024 * 1024) # in MBs
# Asserts that memory usage is atleast 10 times lesser as compared to file size
self.assertLessEqual(max_memory_usage * 10, file_size)
def inner(*args, **kwargs):
fn_kwargs_str = ', '.join(f'{k}={v}' for k, v in kwargs.items())
print(f'\n{fn.__name__}({fn_kwargs_str})')
# Measure time
t = time.perf_counter()
retval = fn(*args, **kwargs)
elapsed = time.perf_counter() - t
print(f'Time {elapsed:0.4}')
# Measure memory
mem, retval = memory_usage((fn, args, kwargs),
retval=True,
timeout=200,
interval=1e-7)
print(f'Memory {max(mem) - min(mem)}')
return retval
def profile(nrange, func):
res = pd.DataFrame(index=nrange, columns=["Time [s]", "Memory Usage"])
for N in res.index:
start = time()
func(N)
end = time()
duration = end - start
memory = memory_usage((func, (N,)))
res.loc[N] = duration, max(memory)
return res
def test_memory_use():
"""Naive test that assumes memory use will never be more than 120 % of
that for first 50 rows"""
folder = os.path.split(__file__)[0]
src = os.path.join(folder, "files", "very_large.xlsx")
wb = openpyxl.load_workbook(src, use_iterators=True)
ws = wb.get_active_sheet()
initial_use = None
for n, line in enumerate(ws.iter_rows()):
if n % 50 == 0:
use = memory_usage(proc=-1, interval=1)[0]
if initial_use is None:
initial_use = use
assert use / initial_use < 1.2
print n, use
async def _status(self, ctx: commands.Context):
"""Information about the bot's status."""
uptime = time.time() - self.bot.uptime
hours = uptime / 3600
minutes = (uptime / 60) % 60
seconds = uptime % 60
users = 0
channel = 0
try:
commands_chart = sorted(self.bot.counter.items(),
key=lambda t: t[1],
reverse=False)
top_command = commands_chart.pop()
command_info = f'{sum(self.bot.counter.values())} (Top Command: {top_command[0]} [x{top_command[1]}])'
except IndexError:
command_info = str(sum(self.bot.counter.values()))
bot_member = ctx.message.guild.get_member(self.bot.user.id)
for guild in self.bot.guilds:
users += len(guild.members)
channel += len(guild.channels)
embed = discord.Embed(colour=bot_member.top_role.colour)
embed.add_field(name='Bot Creator',
value=self.bot.owner.name,
inline=False)
embed.add_field(
name='Uptime',
value='{0:.0f} Hours, {1:.0f} Minutes, and {2:.0f} Seconds'.format(
hours, minutes, seconds),
inline=False)
embed.add_field(name='Total Users', value=users)
embed.add_field(name='Total Channels', value=channel)
embed.add_field(name='Total Servers', value=str(len(self.bot.guilds)))
embed.add_field(name='Command Usage', value=command_info)
embed.add_field(name='Bot Version', value=self.bot.version)
embed.add_field(name='Discord.py Version', value=discord.__version__)
embed.add_field(name='Python Version', value=platform.python_version())
embed.add_field(name='Memory Usage',
value='{} MB'.format(round(memory_usage(-1)[0], 3)))
embed.add_field(name='Operating System',
value='{} {} {}'.format(platform.system(),
platform.release(),
platform.version()),
inline=False)
await ctx.send(embed=embed)
def _memory_usage(func, gallery_conf):
"""Get memory usage of a function call."""
if gallery_conf['show_memory']:
from memory_profiler import memory_usage
assert callable(func)
mem, out = memory_usage(func,
max_usage=True,
retval=True,
multiprocess=True)
try:
mem = mem[0] # old MP always returned a list
except TypeError: # 'float' object is not subscriptable
pass
else:
out = func()
mem = 0
return out, mem
def performance_testing(data: Sequence, tests_count: int):
results_times = []
results_memories = []
occurences = []
for batch in data:
times_of_batch = []
memories_of_batch = []
for _ in range(tests_count):
performance_memory, vals = memory_usage(
(aho_corasick, (batch[0], batch[1])), retval=True)
occurrences, performance_time = vals
times_of_batch.append(performance_time)
memories_of_batch.append(
max(performance_memory) - min(performance_memory))
results_times.append(times_of_batch)
results_memories.append(memories_of_batch)
return [results_times, results_memories, occurrences]
def __init__(self):
# keep a global accounting for the last known memory usage
# which is the reference point for the memory delta calculation
self.previous_call_memory_usage = memory_profiler.memory_usage()[0]
self.t1 = time.time() # will be set to current time later
self.keep_watching = True
self.peak_memory_usage = -1
self.peaked_memory_usage = -1
self.memory_delta = 0
self.time_delta = 0
self.watching_memory = True
self.ip = get_ipython()
self.input_cells = self.ip.user_ns['In']
self._measurements = namedtuple(
'Measurements',
['memory_delta', 'time_delta', 'memory_peak', 'memory_usage'],
verbose=False)
async def mProfileTask(self):
try:
from memory_profiler import memory_usage
assert self.app.cmdArgs.memprofiling is True
except (ImportError, Exception):
pass
while not self.should_stop:
await asyncio.sleep(10)
lt = int(self.app.loop.time())
usage = memory_usage(-1, interval=.2, timeout=1)
if usage:
log.debug(
f'Memory Usage (LT: {lt}): {usage[0]}'
)
def __call__(self, trial, train_loader, test_loader, epoch: int, device,
optimizer, scheduler) -> Tuple[float, float]:
if self.stream is None:
self.stream = Profiler.filename.open('w+')
print(f"AdaS: Profiler: Writing csv to {Profiler.filename}")
else:
if self.trial != trial:
self.stream.close()
self.stream = Profiler.filename.open('w+')
print(f"AdaS: Profiler: Writing csv to {Profiler.filename}")
self.trial = trial
self.gpu.update()
self.pr.enable()
result = memory_usage(proc=(self.function,
(trial, train_loader, test_loader, epoch,
device, optimizer, scheduler)),
max_usage=True,
retval=True)
self.pr.disable()
s = io.StringIO()
sortby = SortKey.CUMULATIVE
ps = pstats.Stats(self.pr, stream=s).sort_stats(sortby)
ps.print_stats(
"epoch_iteration|step|trial_iteration|test_main|evaluate")
stats_list = pstats_to_dict(s.getvalue())
header = 'epoch,epoch_gpu_mem_used,epoch_gpu_temp,epoch_ram_used'
content = f'{epoch},{self.gpu.mem_used - Profiler.base_mem_used},{self.gpu.temperature},{result[0]}'
for stat in stats_list:
header += f",{stat['name']}_n_calls"
header += f",{stat['name']}_tot_time"
header += f",{stat['name']}_per_call1"
header += f",{stat['name']}_cum_time"
header += f",{stat['name']}_per_call2"
content += f",{stat['n_calls']}"
content += f",{stat['tot_time']}"
content += f",{stat['per_call1']}"
content += f",{stat['cum_time']}"
content += f",{stat['per_call2']}"
header += '\n'
content += '\n'
if not self.header_written:
self.stream.write(header)
self.header_written = True
self.stream.write(content)
return result[1]
def watch_memory():
import time
# bring in the global memory usage value from the previous iteration
global previous_call_memory_usage, peak_memory_usage, keep_watching, perf_proc
nbr_commands = len(In)
new_memory_usage = memory_profiler.memory_usage()[0]
memory_delta = new_memory_usage - previous_call_memory_usage
keep_watching = False
peaked_memory_usage = max(0, peak_memory_usage - new_memory_usage)
# calculate time delta using global t1 (from the pre-run event) and current
# time
time_delta_secs = time.time() - t1
perf_values = []
if perf_proc:
# pause if necessary to attempt to make sure we get a sample from perf...
# as the 100ms min sample time and flushing oddness means I don't get a
# sample very quickly for short-running tasks
MIN_TIME_TO_GET_PERF_SAMPLE = 0.2
if time_delta_secs < MIN_TIME_TO_GET_PERF_SAMPLE:
print("PAUSING to get perf sample for {}s".format(
MIN_TIME_TO_GET_PERF_SAMPLE))
time.sleep(MIN_TIME_TO_GET_PERF_SAMPLE
) # pause until at least 0.1s has passed
# if we have a valid perf running then capture that information
perf_values = perf_process.finish_perf(perf_proc)
cmd = In[nbr_commands - 1]
# convert the results into a pretty string
#output_template = "'{cmd}' used {memory_delta:0.4f} MiB RAM in {time_delta:0.2f}s, peaked {peaked_memory_usage:0.2f} MiB above current, total RAM usage {memory_usage:0.2f} MiB"
output_template = "Used {memory_delta:0.4f} MiB RAM in {time_delta:0.2f}s, peaked {peaked_memory_usage:0.2f} MiB above current, total RAM usage {memory_usage:0.2f} MiB"
output = output_template.format(time_delta=time_delta_secs,
cmd=cmd,
memory_delta=memory_delta,
peaked_memory_usage=peaked_memory_usage,
memory_usage=new_memory_usage)
print(str(output))
if perf_values:
perf_average = int(sum(perf_values) / float(time_delta_secs))
#print("perf value for {} averages to {:,}/second, raw samples:".format(perf_process.EVENT_TYPE, perf_average), perf_values)
print("perf value for {} averages to {:,}/second".format(
perf_process.EVENT_TYPE, perf_average))
else:
print(
"perf - no results to report, possibly the collection time was too short?"
)
previous_call_memory_usage = new_memory_usage
def run_benchmark(size, density, radius, algorithm, type, seed):
neurons = []
# 47% of neurons in size
neurons.extend(big_sample(msn_d1, 0.475 * density * size**3))
# print('msn_d1 neurons added', len(neurons), 'total')
# 47% of neurons in size
neurons.extend(big_sample(msn_d2, 0.475 * density * size**3))
# print('msn_d2 neurons added', len(neurons), 'total')
# 1% of neurons in size
neurons.extend(big_sample(fs_pv, 0.01 * density * size**3))
# print('fs_pv neurons added', len(neurons), 'total')
# rotate and translate randomly
for n in neurons:
axis = random_axis(n)
theta = random() * 2 * pi
transform = rotation_matrix(axis, theta)
for nn in n:
nn.transform(transform)
# print(len(neurons), 'neurons rotated')
for n in neurons:
x, y, z = [1000 * (random() * size - size / 2) for _ in range(3)]
for nn in n:
nn.translate(x, y, z)
# print(len(neurons), 'neurons translated')
for n in neurons:
for nn in n:
nn.name = uuid4()
# print(len(neurons), 'neurons renamed')
for i in range(10):
print(algorithm.__name__, size, density, radius, sep=',', end=',')
tbefore = time()
ramsamples = memory_usage((algorithm, (neurons, radius)))
print(time() - tbefore,
min(ramsamples),
max(ramsamples),
type,
seed,
sep=',')
def track_repeated_subset_memratio(self, n_obs, n_var, attr_set,
subset_dim, index_kind):
nviews = 5
base_size = get_anndata_memsize(self.adata)
mem_recording = memory_usage(
(
sedate(take_repeated_view, 0.005),
(self.adata, ),
{
"dim": subset_dim,
"index_kind": index_kind,
"nviews": nviews,
"ratio": 0.9,
},
),
interval=0.001,
)
return (np.max(mem_recording) - np.min(mem_recording)) / base_size
def StartMeMonitoring(pid, datasetName):
filename = datasetName + '-memory.txt'
print("Memory Logger Start\n")
print("Result File: " + filename)
"""
if not os.path.exists(os.path.dirname(filename)):
try:
os.makedirs(os.path.dirname(filename))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
"""
f = open(filename, "a")
mem_usage = memory_usage(int(pid),
interval=1,
timeout=3600000,
retval=False,
stream=f)
def profile_memory():
"""Profile the memory usage of the Python program with memory_profiler"""
from memory_profiler import profile
from memory_profiler import memory_usage
set_logger_debug()
rpa_eb = RPA_EB(MODEL_PATH, SHAPE, BUF_SIZE)
# Add decorators for potentially memory eating methods
rpa_eb._run_test = profile(rpa_eb._run_test)
rpa_eb._proc_frame = profile(rpa_eb._proc_frame)
rpa_eb._merge_bboxes = profile(rpa_eb._merge_bboxes)
def test_single_proc():
rpa_eb.run(test=True, multi_process=False, frame_num=1)
test_single_proc()
print("Total memory usage: {} MiB".format(memory_usage()[0]))
def test_main():
min_kb, max_kb = get_bounds()
# This should be the **only** test function here.
gc.disable()
intersect_all()
kb_used_after = memory_profiler.memory_usage(max_usage=True)
if min_kb <= kb_used_after <= max_kb:
status = 0
msg = SUCCESS_TEMPLATE.format(kb_used_after)
print(msg)
else:
status = 1
msg = ERR_TEMPLATE.format(kb_used_after, min_kb, max_kb)
print(msg, file=sys.stderr)
gc.enable()
sys.exit(status)
def Map(L):
# print "Map:", multiprocessing.current_process().name, "\t",
#print len(L)
# save to global and process
# workerLoad.append(memory_usage(-1, interval=.2, timeout=.2).pop())
# workerLoad = measureLoad()
results = {} # key value storage
for line in L:
key = str(line[0] + ":" + line[1] + ":" + line[2])
try:
results[key] += 1
except KeyError:
results[key] = 1
print "Map: ", [sys.getsizeof(L), sys.getsizeof(results)]
return results, [
multiprocessing.current_process().name,
memory_usage(-1, interval=.0001, timeout=.0001).pop()
], [sys.getsizeof(L), sys.getsizeof(results)]
def profile_memory(fn):
'''
Given the function to profile, profiles the memory usage for the function.
Args:
fn (python func): Function to profile
Returns:
Nothing (saves the memory profile of the function recorded at fixed intervals)
'''
logging.info('PROFILING MEMORY ...')
mem_usage = memory_usage(fn, interval=.01)
logging.info('PROFILED MEMORY.')
logging.info('max memory={:3f}MB.'.format(max(mem_usage)))
pickle.dump(mem_usage, open('memory_example.out', 'wb'))
def inner(*args, **kwargs):
fn_kwargs_str = ", ".join(f"{k}={v}" for k, v in kwargs.items())
print(f"\n{fn.__name__}({fn_kwargs_str})")
# Measure time
t = time.perf_counter()
return_value = fn(*args, **kwargs)
elapsed = time.perf_counter() - t
print(f"Time spent: {elapsed:0.4}")
# Measure memory
mem, return_value = memory_usage((fn, args, kwargs),
retval=True,
timeout=200,
interval=1e-7)
print(f"Memory used: {max(mem) - min(mem)}")
return return_value
def inner(*args, **kwargs):
#fn_kwargs_str = ', '.join(f'{k}={v}' for k, v in kwargs.items())
# if you want to print the function name and kwargs - print(f'\n{fn.__name__}({fn_kwargs_str})')
# Measure time and memory
# set time start
t = time.perf_counter()
# execute function and capture memory usage
mem, retval = memory_usage((fn, args, kwargs), retval=True, interval=1e-7) #timeout=200
elapsed = time.perf_counter() - t
# print results of time and memory
print(f'Time {elapsed:0.4}')
print(f'Memory {max(mem) - min(mem)}')
# return function output
return retval
