def test_memory_usage_coordinates():
"""
Watch out for high memory usage on huge spatial files
"""
ntf = tempfile.NamedTemporaryFile()
tracemalloc.start()
snap1 = tracemalloc.take_snapshot()
# create a "flat" cube
cube,_ = utilities.generate_gaussian_cube(shape=[1,2000,2000])
sz = _.dtype.itemsize
snap1b = tracemalloc.take_snapshot()
diff = snap1b.compare_to(snap1, 'lineno')
diffvals = np.array([dd.size_diff for dd in diff])
# at this point, the generated cube should still exist in memory
assert diffvals.max()*u.B >= 2000**2*sz*u.B
del _
snap2 = tracemalloc.take_snapshot()
diff = snap2.compare_to(snap1b, 'lineno')
assert diff[0].size_diff*u.B < -0.3*u.MB
print(cube)
# printing the cube should not occupy any more memory
# (it will allocate a few bytes for the cache, but should *not*
# load the full 2000x2000 coordinate arrays for RA, Dec
snap3 = tracemalloc.take_snapshot()
diff = snap3.compare_to(snap2, 'lineno')
assert sum([dd.size_diff for dd in diff])*u.B < 100*u.kB
def req_debug_handler(self):
global mem_stat
req = urlparse.urlparse(self.path).query
reqs = urlparse.parse_qs(req, keep_blank_values=True)
try:
import tracemalloc
import gc
gc.collect()
if not mem_stat or "reset" in reqs:
mem_stat = tracemalloc.take_snapshot()
snapshot = tracemalloc.take_snapshot()
if "compare" in reqs:
top_stats = snapshot.compare_to(mem_stat, 'traceback')
else:
top_stats = snapshot.statistics('traceback')
python_lib = os.path.join(root_path, "python27")
dat = ""
for stat in top_stats[:100]:
print("%s memory blocks: %.1f KiB" % (stat.count, stat.size / 1024))
lines = stat.traceback.format()
ll = "\n".join(lines)
ln = len(lines)
pl = ""
for i in xrange(ln, 0, -1):
line = lines[i - 1]
print(line)
if line[8:].startswith(python_lib):
break
if not line.startswith(" File"):
pl = line
continue
if not line[8:].startswith(root_path):
break
ll = line[8:] + "\n" + pl
if ll[0] == "[":
pass
dat += "%d KB, count:%d %s\n" % (stat.size / 1024, stat.count, ll)
if hasattr(threading, "_start_trace"):
dat += "\n\nThread stat:\n"
for path in threading._start_trace:
n = threading._start_trace[path]
if n <= 1:
continue
dat += "%s => %d\n\n" % (path, n)
self.send_response("text/plain", dat)
except Exception as e:
xlog.exception("debug:%r", e)
self.send_response("text/html", "no mem_top")
def main():
# Parse command line options
parser = argparse.ArgumentParser()
parser.add_argument('input_file', help='input ninja file')
parser.add_argument('--encoding', default='utf-8',
help='ninja file encoding')
parser.add_argument('--ninja-deps', help='.ninja_deps file')
args = parser.parse_args()
if DEBUG_ALLOC:
tracemalloc.start(25)
tc_start = tracemalloc.take_snapshot()
# Parse ninja file
manifest = Parser().parse(args.input_file, args.encoding, args.ninja_deps)
if DEBUG_ALLOC:
tc_end = tracemalloc.take_snapshot()
for rule in manifest.rules:
print('rule', rule.name)
for build in manifest.builds:
print('build')
for path in build.explicit_outs:
print(' explicit_out:', path)
for path in build.implicit_outs:
print(' implicit_out:', path)
for path in build.explicit_ins:
print(' explicit_in:', path)
for path in build.implicit_ins:
print(' implicit_in:', path)
for path in build.prerequisites:
print(' prerequisites:', path)
for path in build.depfile_implicit_ins:
print(' depfile_implicit_in:', path)
for pool in manifest.pools:
print('pool', pool.name)
for default in manifest.defaults:
print('default')
for path in default.outs:
print(' out:', path)
if DEBUG_ALLOC:
top_stats = tc_end.compare_to(tc_start, 'traceback')
with open('tracemalloc.log', 'w') as fp:
for s in top_stats:
print('', file=fp)
print('========================================', file=fp)
print(s, file=fp)
for line in s.traceback.format():
print(line, file=fp)
def measure_memory_diff(self, func):
import tracemalloc
tracemalloc.start()
try:
before = tracemalloc.take_snapshot()
# Keep the result and only delete it after taking a snapshot
res = func()
after = tracemalloc.take_snapshot()
del res
return after.compare_to(before, 'lineno')
finally:
tracemalloc.stop()
def memory_obj(args: tuple, packet: ircp.Packet, ___: dict):
""" Print the biggest memory hogs """
if not _IS_TRACING:
return packet.notice(
"Sorry, but tracing is currently disabled. "
'Please restart probot with the "PYTHONTRACEMALLOC=NFRAME" '
"environment variable."
)
snapshot = tracemalloc.take_snapshot()
top_stats = snapshot.statistics("filename")
num = 0
if len(args) >= 2:
try:
num = int(args[1])
except ValueError:
return packet.notice("Your argument must be an integer")
else:
return packet.notice("You must specify an object to inspect!")
if len(top_stats) >= num:
output = [packet.notice("Memory hog #{}".format(num))]
obj = top_stats[num]
trace = tracemalloc.get_object_traceback(obj)
for line in trace:
output.append(packet.notice(line))
return output
else:
return packet.notice("Sorry, but that object does not exist")
def flush_and_send(self, cursor = None, wal_end = None):
if os.getenv("DEBUG_MODE") == "yes":
snapshot = tracemalloc.take_snapshot()
display_top(snapshot)
if len(self.batch_msgs) > 0:
comp_batch_msgs = zlib.compress(json.dumps(self.batch_msgs).encode('utf-8'))
if os.getenv("DEBUG_MODE") == "yes" or os.getenv("ROW_LEVEL_MODE") == "yes":
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") + " : sending batch, length: " + str(len(self.batch_msgs)))
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") + " : uncompressed batch, length: " + str(len(json.dumps(self.batch_msgs))))
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") + " : compressed batch, length: " + str(len(comp_batch_msgs)))
sending_not_successfull = True
while sending_not_successfull:
try:
requests.post(self.rest_api_endpoint, headers=self.headers, data=comp_batch_msgs)
sending_not_successfull = False
except requests.exceptions.ConnectionError:
print("can't send POST Request. Retrying in 2 seconds ...")
time.sleep(2)
if os.getenv("DEBUG_MODE") == "yes" or os.getenv("ROW_LEVEL_MODE") == "yes":
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") + " : sending batch successfull ...")
self.batch_msgs.clear()
if cursor is not None and wal_end is not None:
try:
if os.getenv("DEBUG_MODE") == "yes" or os.getenv("ROW_LEVEL_MODE") == "yes":
print("syncing with pg server single")
cursor.send_replication_feedback(flush_lsn=wal_end)
except AttributeError:
print("cannot sync wal_log, since no valuable cursor was provided ...")
signal.alarm(TIMEOUT)
def wrapper(*args, **kwargs):
import sys
do_prof, do_tracemalloc, do_traceopen = 3 * [False]
if len(sys.argv) > 1:
do_prof = sys.argv[1] == "prof"
do_tracemalloc = sys.argv[1] == "tracemalloc"
do_traceopen = sys.argv[1] == "traceopen"
if do_prof or do_tracemalloc or do_traceopen: sys.argv.pop(1)
if do_prof:
print("Entering profiling mode...")
import pstats, cProfile, tempfile
prof_file = kwargs.pop("prof_file", None)
if prof_file is None:
_, prof_file = tempfile.mkstemp()
print("Profiling data stored in %s" % prof_file)
sortby = kwargs.pop("sortby", "time")
cProfile.runctx("main()", globals(), locals(), prof_file)
s = pstats.Stats(prof_file)
s.strip_dirs().sort_stats(sortby).print_stats()
return 0
elif do_tracemalloc:
print("Entering tracemalloc mode...")
# Requires py3.4
try:
import tracemalloc
except ImportError:
print("Error while trying to import tracemalloc (requires py3.4)")
raise SystemExit(1)
tracemalloc.start()
retcode = main(*args, **kwargs)
snapshot = tracemalloc.take_snapshot()
top_stats = snapshot.statistics('lineno')
n = min(len(top_stats), 20)
print("[Top %d]" % n)
for stat in top_stats[:20]:
print(stat)
elif do_traceopen:
try:
import psutil
except ImportError:
print("traceopen requires psutil module")
raise SystemExit(1)
import os
p = psutil.Process(os.getpid())
retcode = main(*args, **kwargs)
print("open_files", p.open_files())
else:
retcode = main(*args, **kwargs)
return retcode
def test(self):
timings = []
memory_usage = []
tracemalloc.start()
for i in range(self.repeat):
before_memory = tracemalloc.take_snapshot()
start_time = time.time()
self.bench()
end_time = time.time()
after_memory = tracemalloc.take_snapshot()
timings.append(end_time - start_time)
memory_usage.append(sum([t.size for t in after_memory.compare_to(before_memory, 'filename')]))
print("time min:", min(timings), "max:", max(timings), "avg:", sum(timings) / len(timings)) # NOQA
print("memory min:", min(memory_usage), "max:", max(memory_usage), "avg:", sum(memory_usage) / len(memory_usage)) # NOQA
def tracemalloc_tool():
# .. cross-platform but but requires Python 3.4 or higher ..
stat = next(filter(lambda item: str(item).startswith(filename),
tracemalloc.take_snapshot().statistics('filename')))
mem = stat.size / _TWO_20
if timestamps:
return mem, time.time()
else:
return mem
def take_snapshot():
snapshot = tracemalloc.take_snapshot()
return snapshot.filter_traces(
(
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "tracemalloc"),
tracemalloc.Filter(False, "<unknown>"),
)
)
def sigterm_handler(signal, frame):
snapshot = tracemalloc.take_snapshot()
top_stats = snapshot.statistics('lineno')
if tracemallocfile:
with open(tracemallocfile, 'w+') as f:
f.write("SIGTERM tracemalloc before shutdown [ Top 10 ]\n")
for stat in top_stats[:10]:
f.write("{}\n".format(stat))
sys.exit(0)
def display_biggest_traceback():
snapshot = tracemalloc.take_snapshot()
top_stats = snapshot.statistics('traceback')
# pick the biggest memory block
stat = top_stats[0]
print("%s memory blocks: %.1f KiB" % (stat.count, stat.size / 1024))
for line in stat.traceback.format():
print(line)
def take_snapshots():
all_snapshots = []
for loop in range(NGET_SNAPSHOT):
objs = [alloc_object() for index in range(NOBJECTS)]
snapshot = tracemalloc.take_snapshot()
objs = None
all_snapshots.append(snapshot)
snapshots = None
all_snapshots = None
def test_snapshot_save_attr(self):
# take a snapshot with a new attribute
snapshot = tracemalloc.take_snapshot()
snapshot.test_attr = "new"
snapshot.dump(support.TESTFN)
self.addCleanup(support.unlink, support.TESTFN)
# load() should recreate the attribute
snapshot2 = tracemalloc.Snapshot.load(support.TESTFN)
self.assertEqual(snapshot2.test_attr, "new")
def test_does_not_leak_too_much(self):
tracemalloc.start()
gc.collect()
series = []
snapshot1 = tracemalloc.take_snapshot()
for i in range(100):
try:
execute_script(self.feature, self)
except Exception:
pass
gc.collect()
snapshot2 = tracemalloc.take_snapshot()
stats = snapshot2.compare_to(snapshot1, "lineno")
snapshot1 = snapshot2
series.append(sum(stat.size / 1024 for stat in stats))
tracemalloc.stop()
series = series[1:] # ignore first run, which creates regex
cv = statistics.stdev(series) / statistics.mean(series)
assert cv < 0.1
async def test_leak_in_transport(zipkin_url, client, loop):
tracemalloc.start()
endpoint = az.create_endpoint('simple_service')
tracer = await az.create(zipkin_url, endpoint, sample_rate=1,
send_interval=0.0001, loop=loop)
await asyncio.sleep(5)
gc.collect()
snapshot1 = tracemalloc.take_snapshot()
await asyncio.sleep(10)
gc.collect()
snapshot2 = tracemalloc.take_snapshot()
top_stats = snapshot2.compare_to(snapshot1, 'lineno')
count = sum(s.count for s in top_stats)
await tracer.close()
assert count < 400 # in case of leak this number is around 901452
def _stop_memory_tracing():
try:
import tracemalloc
except ImportError:
return
snapshot = tracemalloc.take_snapshot()
_log_memory_top(snapshot)
tracemalloc.stop()
def print_malloc_context(**kwargs):
"""
:param \**kwargs: see print_malloc_snapshot
"""
if tracemalloc is None:
logger.error('tracemalloc required')
return
tracemalloc.start()
yield
snapshot = tracemalloc.take_snapshot()
print_malloc_snapshot(snapshot, **kwargs)
def test_snapshot(self):
obj, source = allocate_bytes(123)
# take a snapshot
snapshot = tracemalloc.take_snapshot()
# write on disk
snapshot.dump(support.TESTFN)
self.addCleanup(support.unlink, support.TESTFN)
# load from disk
snapshot2 = tracemalloc.Snapshot.load(support.TESTFN)
self.assertEqual(snapshot2.traces, snapshot.traces)
# tracemalloc must be tracing memory allocations to take a snapshot
tracemalloc.stop()
with self.assertRaises(RuntimeError) as cm:
tracemalloc.take_snapshot()
self.assertEqual(str(cm.exception),
"the tracemalloc module must be tracing memory "
"allocations to take a snapshot")
def dump_tracemalloc():
"""
Dumps memory usage information to file
"""
gc.collect()
snapshot = tracemalloc.take_snapshot()
output_file = PROFILING_OUTPUT_FMT % get_filename_fmt()
with open(output_file, 'wb') as fp:
cPickle.dump(snapshot, fp, 2)
# Make sure the snapshot goes away
snapshot = None
def tracemalloc_dump() -> None:
if not tracemalloc.is_tracing():
logger.warning("pid {}: tracemalloc off, nothing to dump"
.format(os.getpid()))
return
# Despite our name for it, `timezone_now` always deals in UTC.
basename = "snap.{}.{}".format(os.getpid(),
timezone_now().strftime("%F-%T"))
path = os.path.join(settings.TRACEMALLOC_DUMP_DIR, basename)
os.makedirs(settings.TRACEMALLOC_DUMP_DIR, exist_ok=True)
gc.collect()
tracemalloc.take_snapshot().dump(path)
procstat = open('/proc/{}/stat'.format(os.getpid()), 'rb').read().split()
rss_pages = int(procstat[23])
logger.info("tracemalloc dump: tracing {} MiB ({} MiB peak), using {} MiB; rss {} MiB; dumped {}"
.format(tracemalloc.get_traced_memory()[0] // 1048576,
tracemalloc.get_traced_memory()[1] // 1048576,
tracemalloc.get_tracemalloc_memory() // 1048576,
rss_pages // 256,
basename))
def stop_profiler(self):
self.agent.log('Deactivating memory allocation profiler.')
with self.profile_lock:
if self.overhead_monitor:
self.overhead_monitor.cancel()
self.overhead_monitor = None
if tracemalloc.is_tracing():
snapshot = tracemalloc.take_snapshot()
self.agent.log('Allocation profiler memory overhead {0} bytes'.format(tracemalloc.get_tracemalloc_memory()))
tracemalloc.stop()
self.process_snapshot(snapshot, time.time() - self.start_ts)
def showMemoryTrace():
try:
import tracemalloc # @UnresolvedImport
except ImportError:
pass
else:
snapshot = tracemalloc.take_snapshot()
stats = snapshot.statistics("lineno")
printLine("Top 50 memory allocations:")
for count, stat in enumerate(stats):
if count == 50:
break
printLine(stat)
def trace_print():
global snapshot
snapshot2 = tracemalloc.take_snapshot()
snapshot2 = snapshot2.fileter_traces(
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
tracemalloc.Filter(False, tracemalloc.__file__)
)
if snapshot2 is not None:
print("="*10, " Begin Trace:")
top_stats = snapshot2.compare_to(snapshot, "lineno", cumulative=True)
for stat in top_stats[:10]:
print(stat)
snapshot = snapshot2
def memprofile():
import resource
import tracemalloc
tracemalloc.start()
ast = parse_file('/tmp/197.c')
print('Memory usage: %s (kb)' %
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
snapshot = tracemalloc.take_snapshot()
print("[ tracemalloc stats ]")
for stat in snapshot.statistics('lineno')[:20]:
print(stat)
def take_snapshot(self):
filename = (filename_pattern
% (os.getpid(), self.counter))
t0 = time.time()
print("Write snapshot into %s..." % filename, file=sys.__stderr__)
gc.collect()
snapshot = tracemalloc.take_snapshot()
with open(filename, "wb") as fp:
# Pickle version 2 can be read by Python 2 and Python 3
pickle.dump(snapshot, fp, 2)
snapshot = None
dt = time.time() - t0
print("Snapshot written into %s (%.1f sec)" % (filename, dt),
file=sys.__stderr__)
self.counter += 1
def snapshot(filename=None):
with _lock:
if not is_running():
logging.error('Memory profiler must be running '
'to take snapshots')
return
name = filename or _make_snapshot_name()
gc.collect()
snap = tracemalloc.take_snapshot()
with gzip.open(name + '.gz', 'wb') as fp:
# Pickle version 2 can be read by Python 2 and Python 3
pickle.dump(snap, fp, 2)
snap = None
return name
def test_create_snapshot(self):
raw_traces = [(5, (("a.py", 2),))]
with contextlib.ExitStack() as stack:
stack.enter_context(patch.object(tracemalloc, "is_tracing", return_value=True))
stack.enter_context(patch.object(tracemalloc, "get_traceback_limit", return_value=5))
stack.enter_context(patch.object(tracemalloc, "_get_traces", return_value=raw_traces))
snapshot = tracemalloc.take_snapshot()
self.assertEqual(snapshot.traceback_limit, 5)
self.assertEqual(len(snapshot.traces), 1)
trace = snapshot.traces[0]
self.assertEqual(trace.size, 5)
self.assertEqual(len(trace.traceback), 1)
self.assertEqual(trace.traceback[0].filename, "a.py")
self.assertEqual(trace.traceback[0].lineno, 2)
def trace(self, signum, frame): # pylint: disable=unused-argument
""" Signal handler used to take snapshots of the running process. """
# the last pending signal after trace_stop
if not self.profiling:
return
gc.collect()
snapshot = tracemalloc.take_snapshot()
timestamp = time.time()
sample_data = (timestamp, snapshot)
# *Must* use the HIGHEST_PROTOCOL, otherwise the serialization will
# use GBs of memory
pickle.dump(sample_data, self.trace_stream, protocol=pickle.HIGHEST_PROTOCOL)
self.trace_stream.flush()
def run(self):
if hasattr(signal, 'pthread_sigmask'):
# Available on UNIX with Python 3.3+
signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG))
while True:
logger.debug('Sleeping {0} secongs...'.format(self.interval))
time.sleep(self.interval)
filename = ("/tmp/tracemalloc-%d-%04d.dump"
% (os.getpid(), self.counter))
logger.info("Write snapshot into %s..." % filename)
gc.collect()
snapshot = tracemalloc.take_snapshot()
snapshot.dump(filename )
self.snapshot_q.put(filename)
logger.debug('Queue size: {0}'.format(self.snapshot_q.qsize()))
snapshot = None
logger.info("Snapshot written into %s" % filename)
self.counter += 1
def _run(**kwargs):
# Make sure that, if one worker crashes, the entire MPI process is aborted
def handle_exception(exc_type, exc_value, exc_traceback):
sys.__excepthook__(exc_type, exc_value, exc_traceback)
sys.stderr.flush()
if hvd.size() > 1:
mpi.COMM_WORLD.Abort(1)
sys.excepthook = handle_exception
track_memory = kwargs['trace_memory']
disable_logging = bool(kwargs['disable_logging'])
warm_up_cycles = kwargs['warm_up_cycles']
log_after_warm_up = kwargs['log_after_warm_up']
screenshot_merge = kwargs['screenshot_merge']
clear_checkpoints = list(
filter(None, kwargs['clear_checkpoints'].split(':')))
if 'all' in clear_checkpoints:
clear_checkpoints = CHECKPOINT_ABBREVIATIONS.keys()
if track_memory:
tracemalloc.start(25)
kwargs = del_out_of_setup_args(kwargs)
expl, log_par = setup(**kwargs)
local_logger.info('setup done')
# We only need one MPI worker to log the results
local_logger.info('Initializing logger')
logger = None
traj_logger = None
if hvd.rank() == 0 and not disable_logging:
logger = SimpleLogger(log_par.base_path + '/log.txt')
traj_logger = SimpleLogger(log_par.base_path + '/traj_log.txt')
########################
# START THE EXPERIMENT #
########################
local_logger.info('Starting experiment')
checkpoint_tracker = CheckpointTracker(log_par, expl)
prev_checkpoint = None
merged_dict = {}
sil_trajectories = []
if screenshot_merge[0:9] == 'from_dir:':
screen_shot_dir = screenshot_merge[9:]
else:
screen_shot_dir = f'{log_par.base_path}/screen_shots'
local_logger.info('Initiate cycle')
expl.init_cycle()
local_logger.info('Initiating Cycle done')
if kwargs['expl_state'] is not None:
local_logger.info('Performing warm up cycles...')
expl.start_warm_up()
for i in range(warm_up_cycles):
if hvd.rank() == 0:
local_logger.info(f'Running warm up cycle: {i}')
expl.run_cycle()
expl.end_warm_up()
checkpoint_tracker.n_iters = expl.cycles
checkpoint_tracker.log_warmup = log_after_warm_up
local_logger.info('Performing warm up cycles... done')
while checkpoint_tracker.should_continue():
# Run one iteration
if hvd.rank() == 0:
local_logger.info(f'Running cycle: {checkpoint_tracker.n_iters}')
checkpoint_tracker.pre_cycle()
expl.run_cycle()
checkpoint_tracker.post_cycle()
write_checkpoint = None
if hvd.rank() == 0:
write_checkpoint = checkpoint_tracker.calc_write_checkpoint()
write_checkpoint = mpi.get_comm_world().bcast(write_checkpoint, root=0)
checkpoint_tracker.set_should_write_checkpoint(write_checkpoint)
# Code that should be executed by all workers at a checkpoint generation
if checkpoint_tracker.should_write_checkpoint():
local_logger.debug(
f'Rank: {hvd.rank()} is exchanging screenshots for checkpoint: {expl.frames_compute}'
)
screenshots = expl.trajectory_gatherer.env.recursive_getattr(
'rooms')
if screenshot_merge == 'mpi':
screenshots = flatten_lists(
mpi.COMM_WORLD.allgather(screenshots))
merged_dict = {}
for screenshot_dict in screenshots:
for key, value in screenshot_dict.items():
if key not in merged_dict:
merged_dict[key] = value
else:
after_threshold_screenshot_taken_merged = merged_dict[
key][0]
after_threshold_screenshot_taken_current = screenshot_dict[
key][0]
if after_threshold_screenshot_taken_current and not after_threshold_screenshot_taken_merged:
merged_dict[key] = value
if screenshot_merge == 'disk':
for key, value in merged_dict.items():
filename = f'{screen_shot_dir}/{key}_{hvd.rank()}.png'
os.makedirs(screen_shot_dir, exist_ok=True)
if not os.path.isfile(filename):
im = Image.fromarray(value[1])
im.save(filename)
im_array = imageio.imread(filename)
assert (im_array == value[1]).all()
mpi.COMM_WORLD.barrier()
local_logger.debug('Merging SIL trajectories')
sil_trajectories = [expl.prev_selected_traj]
if hvd.size() > 1:
sil_trajectories = flatten_lists(
mpi.COMM_WORLD.allgather(sil_trajectories))
local_logger.debug(
f'Rank: {hvd.rank()} is done merging trajectories for checkpoint: {expl.frames_compute}'
)
expl.sync_before_checkpoint()
local_logger.debug(
f'Rank: {hvd.rank()} is done synchronizing for checkpoint: {expl.frames_compute}'
)
# Code that should be executed only by the master
if hvd.rank() == 0 and not disable_logging:
gatherer = expl.trajectory_gatherer
return_success_rate = -1
if gatherer.nb_return_goals_chosen > 0:
return_success_rate = gatherer.nb_return_goals_reached / gatherer.nb_return_goals_chosen
exploration_success_rate = -1
if gatherer.nb_exploration_goals_chosen > 0:
exploration_success_rate = gatherer.nb_exploration_goals_reached / gatherer.nb_exploration_goals_chosen
cum_success_rate = 0
for reached in expl.archive.cells_reached_dict.values():
success_rate = sum(reached) / len(reached)
cum_success_rate += success_rate
mean_success_rate = cum_success_rate / len(expl.archive.archive)
logger.write('it', checkpoint_tracker.n_iters)
logger.write('score', expl.archive.max_score)
logger.write('cells', len(expl.archive.archive))
logger.write('ret_suc', return_success_rate)
logger.write('exp_suc', exploration_success_rate)
logger.write('rew_mean', gatherer.reward_mean)
logger.write('len_mean', gatherer.length_mean)
logger.write('ep', gatherer.nb_of_episodes)
logger.write('arch_suc', mean_success_rate)
logger.write('cum_suc', cum_success_rate)
logger.write('frames', expl.frames_compute)
if len(gatherer.loss_values) > 0:
loss_values = np.mean(gatherer.loss_values, axis=0)
assert len(loss_values) == len(gatherer.model.loss_names)
for (loss_value, loss_name) in zip(loss_values,
gatherer.model.loss_names):
logger.write(loss_name, loss_value)
stored_frames = 0
for traj in expl.archive.cell_trajectory_manager.full_trajectories.values(
):
stored_frames += len(traj)
logger.write('sil_frames', stored_frames)
nb_no_score_cells = len(expl.archive.archive)
for weight in expl.archive.cell_selector.selector_weights:
if hasattr(weight, 'max_score_dict'):
nb_no_score_cells = len(weight.max_score_dict)
logger.write('no_score_cells', nb_no_score_cells)
cells_found_ret = 0
cells_found_rand = 0
cells_found_policy = 0
for cell_key in expl.archive.archive:
cell_info = expl.archive.archive[cell_key]
if cell_info.ret_discovered == global_const.EXP_STRAT_NONE:
cells_found_ret += 1
elif cell_info.ret_discovered == global_const.EXP_STRAT_RAND:
cells_found_rand += 1
elif cell_info.ret_discovered == global_const.EXP_STRAT_POLICY:
cells_found_policy += 1
logger.write('cells_found_ret', cells_found_ret)
logger.write('cells_found_rand', cells_found_rand)
logger.write('cells_found_policy', cells_found_policy)
logger.flush()
traj_manager = expl.archive.cell_trajectory_manager
new_trajectories = sorted(
traj_manager.new_trajectories,
key=lambda t: traj_manager.cell_trajectories[t].frame_finished)
for traj_id in new_trajectories:
traj_info = traj_manager.cell_trajectories[traj_id]
traj_logger.write('it', checkpoint_tracker.n_iters)
traj_logger.write('frame', traj_info.frame_finished)
traj_logger.write('exp_strat', traj_info.exp_strat)
traj_logger.write('exp_new_cells', traj_info.exp_new_cells)
traj_logger.write('ret_new_cells', traj_info.ret_new_cells)
traj_logger.write('score', traj_info.score)
traj_logger.write('total_actions', traj_info.total_actions)
traj_logger.write('id', traj_info.id)
traj_logger.flush()
# Code that should be executed by only the master at a checkpoint generation
if checkpoint_tracker.should_write_checkpoint():
local_logger.info(
f'Rank: {hvd.rank()} is writing checkpoint: {expl.frames_compute}'
)
filename = f'{log_par.base_path}/{expl.frames_compute:0{log_par.n_digits}}'
# Save pictures
if len(log_par.save_pictures) > 0:
if screenshot_merge == 'disk':
for file_name in os.listdir(screen_shot_dir):
if file_name.endswith('.png'):
room = int(file_name.split('_')[0])
if room not in merged_dict:
screen_shot = imageio.imread(
f'{screen_shot_dir}/{file_name}')
merged_dict[room] = (True, screen_shot)
elif screenshot_merge[0:9] == 'from_dir:':
for file_name in os.listdir(screen_shot_dir):
if file_name.endswith('.png'):
room = int(file_name.split('.')[0])
if room not in merged_dict:
screen_shot = imageio.imread(
f'{screen_shot_dir}/{file_name}')
merged_dict[room] = (True, screen_shot)
render_pictures(log_par, expl, filename, prev_checkpoint,
merged_dict, sil_trajectories)
# Save archive state
if log_par.save_archive:
save_state(expl.get_state(), filename + ARCHIVE_POSTFIX)
expl.archive.cell_trajectory_manager.dump(filename +
TRAJ_POSTFIX)
# Save model
if log_par.save_model:
expl.trajectory_gatherer.save_model(filename +
MODEL_POSTFIX)
# Clean up previous checkpoint.
if prev_checkpoint:
for checkpoint_type in clear_checkpoints:
if checkpoint_type in CHECKPOINT_ABBREVIATIONS:
postfix = CHECKPOINT_ABBREVIATIONS[checkpoint_type]
else:
postfix = checkpoint_type
with contextlib.suppress(FileNotFoundError):
local_logger.debug(
f'Removing old checkpoint: {prev_checkpoint + postfix}'
)
os.remove(prev_checkpoint + postfix)
prev_checkpoint = filename
if track_memory:
snapshot = tracemalloc.take_snapshot()
display_top(snapshot)
if PROFILER:
local_logger.info(
f'ITERATION: {checkpoint_tracker.n_iters}')
PROFILER.disable()
PROFILER.dump_stats(filename + '.stats')
PROFILER.enable()
local_logger.info(f'Rank {hvd.rank()} finished experiment')
mpi.get_comm_world().barrier()
tracemalloc.start()
test2()
# print('可回收对象', gc.DEBUG_COLLECTABLE)
# print('不可回收对象', gc.DEBUG_UNCOLLECTABLE)
gc.set_debug(gc.DEBUG_UNCOLLECTABLE)
print(gc.garbage)
# 返回函数的16进制地址
print(hex(id(test2)))
# 返回所有被gc管理的对象
print(len(gc.get_objects()))
# 返回test2直接指向的对象
print(gc.get_referents(test2))
# 返回指向test2的对象
print(gc.get_referrers(test2))
snapshot = tracemalloc.take_snapshot()
top_stats = snapshot.statistics('lineno')
print("[ Top 10 ]")
for stat in top_stats[:10]:
print(stat)
# gc.disable() # 这里是否disable事实上无所谓
# gc.set_debug(gc.DEBUG_COLLECTABLE)
# for _ in range(1):
# test2()
# print(gc.garbage)
# gc.collect()
print("**********")
# time.sleep(5)
def getMemory():
"""
Retorna la memoria alocada en un instante de tiempo
"""
return tracemalloc.take_snapshot()
def get_traced_memory(self):
# Get the traced size in the domain
snapshot = tracemalloc.take_snapshot()
domain_filter = tracemalloc.DomainFilter(True, self.domain)
snapshot = snapshot.filter_traces([domain_filter])
return sum(trace.size for trace in snapshot.traces)
def run_SVMreject(trainname, testname, n):
trainDataPath = '/albona/nobackup/biostat/datasets/singlecell/tabulaMuris_benchmark/' + trainname + '.csv'
trainLabelsPath = '/albona/nobackup/biostat/datasets/singlecell/tabulaMuris_benchmark/' + trainname + '_label.csv'
testDataPath = '/albona/nobackup/biostat/datasets/singlecell/tabulaMuris_benchmark/' + testname + '.csv'
testLabelsPath = '/albona/nobackup/biostat/datasets/singlecell/tabulaMuris_benchmark/' + testname + '_label.csv'
# read the data
train = pd.read_csv(trainDataPath, index_col=0, sep=',')
test = pd.read_csv(testDataPath, index_col=0, sep=',')
y_train = pd.read_csv(trainLabelsPath, header=0, index_col=0, sep=',')
y_train = y_train['x'].ravel()
y_test = pd.read_csv(testLabelsPath, header=0, index_col=0, sep=',')
y_test = y_test['x'].ravel()
truelab = []
pred = []
train = train.transpose()
test = test.transpose()
now = time.time()
tracemalloc.start()
Classifier = LinearSVC()
clf = CalibratedClassifierCV(Classifier)
clf.fit(train, y_train)
snapshot = tracemalloc.take_snapshot()
mem_train = display_top(snapshot)
later = time.time()
time_train = int(later - now)
now = time.time()
tracemalloc.start()
predicted = clf.predict(test)
snapshot = tracemalloc.take_snapshot()
mem_test = display_top(snapshot)
later = time.time()
time_test = int(later - now)
prob = np.max(clf.predict_proba(test), axis=1)
unlabeled = np.where(prob < 0.7)
predicted[unlabeled] = 'Unassigned'
truelab = y_test
pred = predicted
truelab = pd.DataFrame(truelab)
pred = pd.DataFrame(pred)
os.chdir("/dora/nobackup/yuec/scclassify/benchmark/SVMreject/vary_test")
truelab.to_csv(n + "_SVMreject_true.csv", index=False)
pred.to_csv(n + "_SVMreject_pred.csv", index=False)
return mem_train, time_train, mem_test, time_test
# > Traceback where an object was allocated
# > Statistics on allocated memory blocks per filename and per line number: total size, number and average size of allocated memory blocks
# > Compute the differences between two snapshots to detect memory leaks
# To trace most memory blocks allocated by Python, the module should be started as early as possible by setting the PYTHONTRACEMALLOC environment
# variable to 1, or by using -X tracemalloc command line option. The tracemalloc.start() function can be called at runtime to start tracing Python
# memory allocations.
#
# Compute differences.
#
# Take two snapshots and display the differences:
#
import tracemalloc
tracemalloc.start()
# ... start your application ...
snapshot1 = tracemalloc.take_snapshot()
# ... call the function leaking memory ...
snapshot2 = tracemalloc.take_snapshot()
top_stats = snapshot2.compare_to(snapshot1, 'lineno')
print("[ Top 10 differences ]")
for stat in top_stats[:10]:
print(stat)
def get_allocated_memory():
gc.collect()
snapshot = tracemalloc.take_snapshot()
current = snapshot.statistics('filename')
return sum(stat.size for stat in current) / (2**20)
def disable(self):
"""
Stop profiling calls.
"""
self.snapshot = tracemalloc.take_snapshot()
def memorytrace_print():
global tracemalloc_start
snapshot = tracemalloc.take_snapshot()
top_stats = snapshot.compare_to(tracemalloc_start, "lineno")
for stat in top_stats[:10]:
print(stat)
def memorytrace_start():
global tracemalloc_start
if tracemalloc_start == None:
print("Begin memorytrace")
tracemalloc.start()
tracemalloc_start = tracemalloc.take_snapshot()
for ds in data_from.get('ds',[]):
ds['exec_stat']='0:未开始'
for one in config_data.get('vars',[]):
one['exec_stat']='0:未开始'
for one in config_data.get('template_output_act',[]):
one['exec_stat']='0:未开始'
for one in config_data.get('text_tpls',[]):
one['exec_stat']='0:未开始'
if __name__ == '__main__':
import glb
import objgraph
import gc,tracemalloc
tracemalloc.start()
b_snapshot = tracemalloc.take_snapshot()
for i in range(10):
with glb.db_connect() as conn:
with conn.cursor(as_dict=True) as cursor:
cursor.execute("SELECT * FROM zhanbao_tbl WHERE id=4274 order by id asc")
row = cursor.fetchone()
while row:
b1_snapshot = tracemalloc.take_snapshot()
try:
print('worker_no:'+ row['worker_no']+"\t"+ str(row['id']) +" "+ str(tracemalloc.get_traced_memory()))
files_template_exec(row['id'],json.loads(row['config_txt']),row['worker_no'],glb.config['UPLOAD_FOLDER'] ,wx_queue=glb.msg_queue)
print("====================================")
print('worker_no:'+ row['worker_no']+"\t"+ str(row['id']) +" "+ str(tracemalloc.get_traced_memory()))
print("====================================")
snapshot2 = tracemalloc.take_snapshot()
top_stats = snapshot2.compare_to(b1_snapshot, 'lineno')
depending on a threshold: 128 KB by default. The threshold is dynamic nowadays.
Use mallopt(M_MMAP_THRESHOLD, nbytes) to change this threshold.
See also:
* http://pushingtheweb.com/2010/06/python-and-tcmalloc/
* http://sourceware.org/ml/libc-alpha/2006-03/msg00033.html
* http://www.linuxdevcenter.com/pub/a/linux/2006/11/30/linux-out-of-memory.html?page=2
* http://cloudfundoo.wordpress.com/2012/05/18/minor-page-faults-and-dynamic-memory-allocation-in-linux/
"""
import gc
import sys
import tracemalloc
tracemalloc.start()
previous_snapshot = tracemalloc.take_snapshot()
def dump_memory():
if 1:
global previous_snapshot
snapshot = tracemalloc.take_snapshot()
exclude = tracemalloc.Filter(False, tracemalloc.__file__)
snapshot = snapshot.filter_traces([exclude])
top_stats = snapshot.compare_to(previous_snapshot, 'lineno')
previous_snapshot = snapshot
print("[ Top 10 ]")
for stat in top_stats[:10]:
print(stat)
def main(argv):
global csv_output_filename
start1 = datetime.datetime.now()
try:
opts, args = getopt.getopt(argv, "ha:n:s:t:v:p:i:", [
"agent=", "numflows=", "flowsize=", "timesteps=", "interval=",
"proportion=", "iter="
])
except getopt.GetoptError:
print(
'run-experiments.py -a <agent> -n <numflows> -s <flowsize> -t <timesteps> -v <interval> -p <proportion> -i <iter>'
)
sys.exit(2)
agent = None
num_flows = None
flows_size = None
timesteps = None
interval = None
iter = None
proportion = None
for opt, arg in opts:
if opt == '-h':
print(
'run-experiments.py -a <agent> -n <numflows> -s <flowsize> -t <timesteps> -v <interval> -p <proportion> -i <iter>'
)
sys.exit()
elif opt in ("-a", "--agent"):
agent = arg
elif opt in ("-n", "--numflows"):
num_flows = arg
elif opt in ("-s", "--flowsize"):
flows_size = arg
elif opt in ("-t", "--timesteps"):
timesteps = arg
elif opt in ("-i", "--iter"):
iter = arg
elif opt in ("-v", "--interval"):
interval = arg
elif opt in ("-p", "--proportion"):
proportion = arg
csv_output_filename = './output-experiments-app/{0}-{1}_flows-{2}-{3}_steps-{4}_sec-prop_{5}-v_{6}.csv'.format(
agent, num_flows, flows_size, timesteps, interval, proportion, iter)
print(
'Running: agent = {0}, number of flows = {1}, flows size = {2}, timesteps = {3}, interval = {4}, proportion = {5}, iter = {6}'
.format(agent, num_flows, flows_size, timesteps, interval, proportion,
iter))
tracemalloc.start()
start_time = datetime.datetime.now()
if agent == 'F-HET':
wait_time = int(timesteps) * 2
print('wait_time = ', wait_time)
time.sleep(wait_time)
elif agent == 'F' or agent == 'F2':
flow_size_bits = int(flows_size.strip('M')) * 8
wait_time = (int(num_flows) * flow_size_bits / 10) * 4
print('wait_time = ', wait_time)
time.sleep(wait_time)
else:
env, original_env = createVectorizedEnv()
# trainLookAheadAgent(env, agent)
testAgent(env, original_env, agent, timesteps)
time_interval = datetime.datetime.now() - start_time
snapshot = tracemalloc.take_snapshot()
memory_usage = getTopMemoryUsage(snapshot)
output_filename_compcosts = './output-experiments-app/{0}-{1}_flows-{2}-{3}steps-{4}_sec-prop_{5}-v_{6}-compcosts.txt'.format(
agent, num_flows, flows_size, timesteps, interval, proportion,
iter)
with open(output_filename_compcosts, 'w+') as output_file:
output_file.write("%s\n" % time_interval)
output_file.write("%s\n" % memory_usage)
print('Arquivo {0} criado.'.format(output_filename_compcosts))
def run_moana(trainname, testname, n):
DataPath = '/albona/nobackup/biostat/datasets/singlecell/tabulaMuris_benchmark/'+trainname+'.csv'
matrix = ExpMatrix.read_tsv(DataPath, sep = ',')
LabelsPath = '/albona/nobackup/biostat/datasets/singlecell/tabulaMuris_benchmark/'+trainname+'_label.csv'
truelab = pd.read_csv(LabelsPath, header=0,index_col=0, sep=',')
data = ExpMatrix(X = matrix.X, genes = matrix.genes, cells = matrix.cells)
data.genes.name = 'Genes'
data.cells.name = 'Cells'
data.index.name = 'Genes'
data.columns.name = 'Cells'
l = CellAnnVector(cells=data.cells, data=truelab['x'].values)
now = time.time()
tracemalloc.start()
clf = CellTypeClassifier()
clf.fit(matrix = data, cell_labels = l)
snapshot = tracemalloc.take_snapshot()
mem_train = display_top(snapshot)
later = time.time()
time_train = int(later - now)
DataPath = '/albona/nobackup/biostat/datasets/singlecell/tabulaMuris_benchmark/'+testname+'.csv'
matrix = ExpMatrix.read_tsv(DataPath, sep = ',')
data = ExpMatrix(X = matrix.X, genes = matrix.genes, cells = matrix.cells)
data.genes.name = 'Genes'
data.cells.name = 'Cells'
data.index.name = 'Genes'
data.columns.name = 'Cells'
now = time.time()
tracemalloc.start()
predictions = clf.predict(data)
snapshot = tracemalloc.take_snapshot()
mem_test = display_top(snapshot)
later = time.time()
time_test = int(later - now)
predictions = np.asarray(predictions)
pred = pd.DataFrame(predictions)
LabelsPath = '/albona/nobackup/biostat/datasets/singlecell/tabulaMuris_benchmark/'+testname+'_label.csv'
truelab = pd.read_csv(LabelsPath, header=0,index_col=None, sep=',')
os.chdir("/dora/nobackup/yuec/scclassify/benchmark/moanna/vary_celltype")
truelab.to_csv(n + "_moana_True.csv", index = False)
pred.to_csv( n + "_moana_Pred.csv", index = False)
return mem_train, time_train, mem_test, time_test
#!/usr/bin/env PYTHONHASHSEED=1234 python3
# Copyright 2014-2019 Brett Slatkin, Pearson Education Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import tracemalloc
tracemalloc.start(10) # Set stack depth
time1 = tracemalloc.take_snapshot() # Before snapshot
import waste_memory
x = waste_memory.run() # Usage to debug
time2 = tracemalloc.take_snapshot() # After snapshot
stats = time2.compare_to(time1, 'lineno') # Compare snapshots
for stat in stats[:3]:
print(stat)
def assignment_b_suffixarray_2():
# For testing.
class TestNormalizer(Normalizer):
_table = str.maketrans({'Ø': 'O'})
def canonicalize(self, buffer: str) -> str:
return buffer
def normalize(self, token: str) -> str:
return token.upper().translate(self._table)
# For testing.
class TestDocument(Document):
def __init__(self, document_id: int, a: str, b: str):
self._document_id = document_id
self._a = a
self._b = b
def get_document_id(self) -> int:
return self._document_id
def get_field(self, field_name: str, default: str) -> str:
if field_name == "a":
return self._a
if field_name == "b":
return self._b
return default
# For testing.
class TestCorpus(Corpus):
def __init__(self):
self._docs = []
self._docs.append(TestDocument(len(self._docs), "ø o\n\n\nø\n\no", "ø o\nø \no"))
self._docs.append(TestDocument(len(self._docs), "ba", "b bab"))
self._docs.append(TestDocument(len(self._docs), "ø o Ø o", "ø o"))
self._docs.append(TestDocument(len(self._docs), "øO" * 10000, "o"))
self._docs.append(TestDocument(len(self._docs), "cbab o øbab Ø ", "ø o " * 10000))
def __iter__(self):
return iter(self._docs)
def size(self) -> int:
return len(self._docs)
def get_document(self, document_id: int) -> Document:
return self._docs[document_id]
# Run the tests!
for fields in [("b",), ("a", "b")]:
# Create the suffix array over the given set of fields. Measure memory usage. If memory usage is
# excessive, most likely the implementation is copying strings or doing other silly stuff instead
# of working with buffer indices. The naive reference implementation is not in any way optimized,
# and uses about 1.5 MB of memory on this corpus.
tracemalloc.start()
snapshot1 = tracemalloc.take_snapshot()
engine = SuffixArray(TestCorpus(), fields, TestNormalizer(), BrainDeadTokenizer())
snapshot2 = tracemalloc.take_snapshot()
for statistic in snapshot2.compare_to(snapshot1, "filename"):
if statistic.traceback[0].filename == inspect.getfile(SuffixArray):
assert statistic.size_diff < 2000000, f"Memory usage is {statistic.size_diff}"
tracemalloc.stop()
results = []
def process(m):
results.append((m['document'].document_id, m['score']))
expected_results = {
('b',): (
('bab', [(1, 1)]),
('ø o', [(4, 19999), (0, 3), (2, 1)]),
('o O', [(4, 19999), (0, 3), (2, 1)]),
('oooooo', []),
('o o o o', [(4, 19997), (0, 1)]),
),
('a', 'b'): (
('bab', [(1, 1)]),
('ø o', [(4, 20000), (0, 6), (2, 4)]),
('o O', [(4, 20000), (0, 6), (2, 4)]),
('oøØOøO', [(3, 1), ]),
('o o o o', [(4, 19997), (0, 2), (2, 1)]),
)
}
for query, expected in expected_results[fields]:
results.clear()
engine.evaluate(query, {'hit_count': 10}, process)
assert results == expected
def get_allocated_khash_memory():
snapshot = tracemalloc.take_snapshot()
snapshot = snapshot.filter_traces(
(tracemalloc.DomainFilter(True, ht.get_hashtable_trace_domain()), ))
return sum(map(lambda x: x.size, snapshot.traces))
def main():
args = parse_args()
program_start = time.time()
if args.trace_malloc:
tracemalloc.start()
print(
"Virtual Change/Drift Detection - Association Rule Mining using Python."
)
print("Drift Algorithm: {}".format(args.drift_algorithm))
print("Input file: {}".format(args.input))
print("Output file prefix: {}".format(args.output))
print("Training window size: {}".format(args.training_window_size))
print("Minimum confidence: {}".format(args.min_confidence))
print("Minimum support: {}".format(args.min_support))
print("Minimum lift: {}".format(args.min_lift))
if args.fixed_drift_confidence is not None:
print("Fixed drift confidence of: {}".format(
args.fixed_drift_confidence))
print("Tracing memory allocations: {}".format(args.trace_malloc))
print("Save rules: {}".format(args.save_rules))
print("Generating maximal itemsets: {}".format(args.maximal_itemsets))
reader = iter(DatasetReader(args.input))
transaction_num = 0
end_of_last_window = 0
cohort_num = 1
volatility_detector = make_volatility_detector(args)
while True:
window = take(args.training_window_size, reader)
if len(window) == 0:
break
print("")
print("Mining window [{},{}]".format(transaction_num,
transaction_num + len(window)))
end_of_last_window = transaction_num + len(window)
transaction_num += len(window)
print("Running FP-Growth...", flush=True)
start = time.time()
(itemsets, itemset_counts,
num_transactions) = mine_fp_tree(window, args.min_support,
args.maximal_itemsets)
assert (num_transactions == len(window))
duration = time.time() - start
print("FPGrowth mined {} items in {:.2f} seconds".format(
len(itemsets), duration))
print("Generating rules...", flush=True)
start = time.time()
rules = list(
generate_rules(itemsets, itemset_counts, num_transactions,
args.min_confidence, args.min_lift))
duration = time.time() - start
print("Generated {} rules in {:.2f} seconds".format(
len(rules), duration),
flush=True)
if len(rules) == 0:
print("No rules; just noise. Skipping change detection.")
print(
"Consider increasing training window size or lowering minsup/conf."
)
continue
if args.save_rules:
start = time.time()
output_filename = args.output + "." + str(cohort_num)
cohort_num += 1
write_rules_to_file(rules, output_filename)
print("Wrote rules for cohort {} to file {} in {:.2f} seconds".
format(cohort_num, output_filename, duration),
flush=True)
drift_detector = make_drift_detector(args, volatility_detector)
drift_detector.train(window, rules)
# Read transactions until a drift is detected.
for transaction in reader:
transaction_num += 1
drift = drift_detector.check_for_drift(transaction,
transaction_num)
if drift is not None:
print(
"Detected drift of type {} at transaction {}, {} after end of training window"
.format(drift.drift_type, transaction_num,
transaction_num - end_of_last_window))
if drift.hellinger_value is not None:
print(
"Hellinger value: {}, confidence interval: {} ± {} ([{},{}])"
.format(drift.hellinger_value, drift.mean,
drift.confidence,
drift.mean - drift.confidence,
drift.mean + drift.confidence))
# Record the drift in the volatility detector. This is used inside
# the drift detector to help determine how large a confidence interval
# is required when detecting drifts.
if volatility_detector is not None:
volatility_detector.add(transaction_num)
# Break out of the inner loop, we'll jump back up to the top and mine
# a new training window.
break
if len(window) < args.training_window_size:
break
print("\nEnd of stream\n")
duration = time.time() - program_start
print("Total runtime {:.2f} seconds".format(duration))
if args.trace_malloc:
(_, peak_memory) = tracemalloc.get_traced_memory()
tracemalloc_memory = tracemalloc.get_tracemalloc_memory()
print("Peak memory usage: {:.3f} MB".format(peak_memory / 10**6))
print("tracemalloc overhead: {:.3f} MB".format(
(tracemalloc_memory / 10**6)))
print("Peak memory usage minus tracemalloc overhead: {:.3f} MB".format(
(peak_memory - tracemalloc_memory) / 10**6))
snapshot = tracemalloc.take_snapshot()
bytes_allocated = sum(x.size for x in snapshot.traces)
print("Total traced memory allocated: {:.3f} MB".format(
bytes_allocated / 10**6))
tracemalloc.stop()
return 0
def getMemory():
"""
toma una muestra de la memoria alocada en instante de tiempo
"""
return tracemalloc.take_snapshot()
async def wrapper(*args, **kwargs):
snapshot = tracemalloc.take_snapshot()
res = await f(*args, **kwargs)
self.stats[f.__name__] = tracemalloc.take_snapshot().compare_to(
snapshot, 'lineno')
return res
def getMemory():
return tracemalloc.take_snapshot()
def print_stat():
snapshot = tracemalloc.take_snapshot()
stats = snapshot.statistics('lineno')
for s in stats:
print(s)
pass
#!/usr/bin/env python3.8
# top_n.py
import tracemalloc
tracemalloc.start(10) # 스택 깊이 설정
time1 = tracemalloc.take_snapshot() # 이전 스냅샷
import waste_memory
x = waste_memory.run() # Usage to debug
time2 = tracemalloc.take_snapshot() # 이후 스냅샷
stats = time2.compare_to(time1, 'lineno') # 두 스냅샷을 비교
for stat in stats[:3]:
print(stat)
def capture(self, idx=1):
"""捕捉最新的快照到指定的槽位"""
self.__dict__['snap%d' %
idx] = tracemalloc.take_snapshot().filter_traces(
self.filters)
def __enter__(self):
# type: () -> MeasureMemory
self.snapshot = tracemalloc.take_snapshot()
return self
def test_history_performance():
try:
tracemalloc.start()
except:
pass
for _ in range( 3 ):
path = "/tmp/test_performance_%d" % random.randint( 100000, 999999 )
if os.path.exists( path ):
continue
assert not os.path.exists( path ), "Couldn't find an unused name: %s" % path
files = []
try:
day = 24*60*60
dur = 3*day # a few days worth of data
regstps = 0.0,5.0 # 0-5secs between updates
numfiles = dur//day+1 # ~1 file/day, but at least 2
values = {} # Initial register values
regscount = 1000 # Number of different registers
regschanged = 1,10 # From 1-25 registers per row
regsbase = 40001
start = timer()
now = beg = start - dur
linecnt = 0
for e in reversed( range( numfiles )):
f = path + (( '.%d' % e ) if e else '') # 0'th file has no extension
files.append( f )
with logger( f ) as l:
if values:
l.write( values, now=now ); linecnt += 1
while now < beg + len(files) * dur/numfiles:
lst = now
now += random.uniform( *regstps )
assert now >= lst
assert timestamp( now ) >= timestamp( lst ), "now: %s, timestamp(now): %s" % ( now, timestamp( now ))
updates = {}
for _ in range( random.randint( *regschanged )):
updates[random.randint( regsbase, regsbase + regscount - 1 )] = random.randint( 0, 1<<16 - 1 )
values.update( updates )
l.write( updates, now=now ); linecnt += 1
lst = now
now += random.uniform( *regstps )
assert now >= lst
assert timestamp( now ) >= timestamp( lst )
if e:
# Compress .1 onward using a random format; randomly delete origin uncompressed file
# so sometimes both files exist
if random.choice( (True, False, False, False) ):
continue # Don't make a compressed version of some files
fz = f + '.%s' % random.choice( ('gz', 'bz2', 'xz') )
files.append( fz )
with opener( fz, mode='wb' ) as fd:
with open( f, 'rb' ) as rd:
fd.write( rd.read() )
if random.choice( (True, False, False) ):
continue # Don't remove some of the uncompressed files
os.unlink( f )
files.pop( files.index( f ))
logging.warning( "Generated data in %.3fs; lines: %d", timer() - start, linecnt )
# Start somewhere within 0-1% the dur of the beg, forcing the load the look back to
# find the first file. Try to do it all in the next 'playback' second (just to push it to
# the max), in 'chunks' pieces.
historical = timestamp( random.uniform( beg + dur*0/100, beg + dur*1/100 ))
basis = timer()
playback = 2.0 * dur/day # Can sustain ~2 seconds / day of history on a single CPU
chunks = 1000
factor = dur / playback
lookahead = 60.0
duration = None
if random.choice( (True,False) ):
duration = random.uniform( dur * 98/100, dur * 102/100 )
begoff = historical.value - beg
endoff = 0 if duration is None else (( historical.value + duration ) - ( beg + dur ))
logging.warning( "Playback starts at beginning %s %s, duration %s, ends at ending %s %s",
timestamp( beg ), format_offset( begoff, ms=False ),
None if duration is None else format_offset( duration, ms=False, symbols='-+' ),
timestamp( beg + dur ), format_offset( endoff, ms=False ))
ld = loader(
path, historical=historical, basis=basis, factor=factor, lookahead=lookahead, duration=duration )
eventcnt = 0
slept = 0
cur = None
while ld:
once = False
while ld.state < ld.AWAITING or not once:
once = True
upcoming = None
limit = random.randint( 0, 250 )
if random.choice( (True,False) ):
upcoming = ld.advance()
if random.choice( (True,False) ) and cur:
# ~25% of the time, provide an 'upcoming' timestamp that is between the
# current advancing historical time and the last load time.
upcoming-= random.uniform( 0, upcoming.value - cur.value )
cur,events = ld.load( upcoming=upcoming, limit=limit )
eventcnt += len( events )
advance = ld.advance()
offset = advance.value - cur.value
logging.detail( "%s loaded up to %s (%s w/ upcoming %14s); %4d future, %4d values: %4d events / %4d limit" ,
ld, cur, format_offset( offset ),
format_offset( upcoming.value - advance.value ) if upcoming is not None else None,
len( ld.future ), len( ld.values ), len( events ), limit )
logging.warning( "%s loaded up to %s; %3d future, %4d values: %6d events total",
ld, cur, len( ld.future ), len( ld.values ), eventcnt )
try:
snapshot = tracemalloc.take_snapshot()
display_top( snapshot, limit=10 )
except:
pass
time.sleep( playback/chunks )
slept += playback/chunks
elapsed = timer() - basis
eventtps = eventcnt // ( elapsed - slept )
logging.error( "Playback in %.3fs (slept %.3fs); events: %d ==> %d historical records/sec",
elapsed, slept, eventcnt, eventtps )
if not logging.getLogger().isEnabledFor( logging.NORMAL ):
# Ludicrously low threshold, to pass tests on very slow machines
assert eventtps >= 1000, \
"Historical event processing performance low: %d records/sec" % eventtps
try:
display_biggest_traceback()
except:
pass
except Exception as exc:
logging.normal( "Test failed: %s", exc )
'''
for f in files:
logging.normal( "%s:\n %s", f, " ".join( l for l in open( f )))
'''
raise
finally:
for f in files:
logging.detail( "unlinking %s", f )
try:
os.unlink( f )
except:
pass
profiler.runcall(my_program)
stats = Stats(profiler)
stats.strip_dirs()
stats.sort_stats('cumulative')
stats.print_callers()
################################
# detecting memory leaks
################################
import os
import hashlib
import tracemalloc
tracemalloc.start(10)
time1 = tracemalloc.take_snapshot()
#TODO: to debug .. not working on windows
#import waste_memory
#x = waste_memory.run()
time2 = tracemalloc.take_snapshot()
stats = time2.compare_to(time1, 'lineno')
for stat in stats[:3]:
print(stat)
class Obj(object):
def __init__(self):
self.x = os.urandom(100)
self.y = hashlib.sha1(self.x).hexdigest()
def print_stats(self, path, coverage=None):
''' Prints stats about the generated path
Args:
path: A Nx3 array with waypoints
'''
print("Done with planning. Calculating stats.")
if not len(path):
self.print("ERROR: Length of path is 0.")
end_time = timeit.default_timer()
snapshot = tracemalloc.take_snapshot()
nbr_of_points_in_path = len(path)
def get_memory_consumption(snapshot, key_type='lineno'):
''' Calculates memory consumption of the algorithm in KiB
'''
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
total = sum(stat.size for stat in top_stats)
return total / 1024
def get_length_of_path(path):
''' Calculates length of the path in meters
'''
length = 0
for point_idx in range(len(path) - 1):
length += np.linalg.norm(path[point_idx] - path[point_idx + 1])
return length
def get_total_rotation(path):
''' Calculates the total rotation made by the robot while executing the path
'''
rotation = 0
for point_idx in range(len(path) - 2):
prev = (path[point_idx + 1] -
path[point_idx]) / np.linalg.norm(path[point_idx] -
path[point_idx + 1])
next = (path[point_idx + 2] - path[point_idx + 1]
) / np.linalg.norm(path[point_idx + 2] -
path[point_idx + 1])
dot_product = np.dot(prev, next)
curr_rotation = np.arccos(dot_product)
if not np.isnan(curr_rotation):
rotation += abs(curr_rotation)
return rotation
length_of_path = get_length_of_path(path)
rotation = get_total_rotation(path[:, 0:2])
#unessecary_coverage_mean = self.coverable_pcd.get_coverage_count_per_point(path)
computational_time = end_time - self.start_time
if coverage is None:
coverage = self.coverable_pcd.get_coverage_efficiency()
memory_consumption = get_memory_consumption(snapshot)
print_text = "\n" + "=" * 20
print_text += "\nAlgorithm: " + self.name
print_text += "\nCoverage efficiency: " + str(round(coverage * 100,
2)) + "%"
print_text += "\nNumber of waypoints: " + str(nbr_of_points_in_path)
print_text += "\nLength of path: " + str(
round(length_of_path)) + " meter"
print_text += "\nTotal rotation: " + str(round(rotation)) + " rad"
#print_text += "\nVisits per point: " + str(unessecary_coverage_mean)
print_text += "\nComputational time: " + str(
round(computational_time, 1)) + " sec"
print_text += "\nMemory consumption: " + str(
round(memory_consumption, 1)) + " KiB"
print_text += "\n" + "=" * 20
self.print(print_text)
return {
"Algorithm": self.name,
"Coverage efficiency": round(coverage * 100, 2),
"Number of waypoints": nbr_of_points_in_path,
"Length of path": round(length_of_path),
"Total rotation": round(rotation),
#"Visits per point": unessecary_coverage_mean,
"Computational time": round(computational_time, 1),
"Memory consumption": round(memory_consumption)
}
import tracemalloc as t
print("*start")
print([t._format_size(x, False) for x in t.get_traced_memory()])
t.start()
L = [[_ for _ in range(10000)] for i in range(100)]
print("*gen")
print([t._format_size(x, False) for x in t.get_traced_memory()])
snapshot = t.take_snapshot()
for stats in snapshot.statistics("traceback")[:3]:
print(stats)
print("----------------------------------------")
snapshot = t.take_snapshot()
for stats in snapshot.statistics("lineno", cumulative=True)[:3]:
print(stats)
t.stop()
print([t._format_size(x, False) for x in t.get_traced_memory()])
async def debug(self, ctx):
if tracemalloc.is_tracing():
snapshot = tracemalloc.take_snapshot()
top_stats = snapshot.statistics("lineno")
await ctx.send("```" + "\n".join([str(x) for x in top_stats[:10]]) + "```")
