def monitor_overhead():
if tracemalloc.is_tracing() and tracemalloc.get_tracemalloc_memory(
) > self.MAX_MEMORY_OVERHEAD:
logger.debug(
'Allocation sampler memory overhead limit exceeded: %s bytes',
tracemalloc.get_tracemalloc_memory())
self.stop_sampler()
def monitor_overhead():
if tracemalloc.is_tracing() and tracemalloc.get_tracemalloc_memory(
) > self.MAX_MEMORY_OVERHEAD:
self.agent.log(
'Allocation profiler memory overhead limit exceeded: {0} bytes'
.format(tracemalloc.get_tracemalloc_memory()))
self.stop_profiler()
def record(self, max_duration):
if self.agent.config.is_profiling_disabled():
return
self.agent.log('Activating memory allocation profiler.')
def start():
tracemalloc.start(self.MAX_TRACEBACK_SIZE)
self.agent.run_in_main_thread(start)
duration = 0
step = 1
while duration < max_duration:
time.sleep(step)
duration += step
if tracemalloc.get_tracemalloc_memory() > self.MAX_MEMORY_OVERHEAD:
break
self.agent.log('Deactivating memory allocation profiler.')
if tracemalloc.is_tracing():
snapshot = tracemalloc.take_snapshot()
self.agent.log(
'Allocation profiler memory overhead {0} bytes'.format(
tracemalloc.get_tracemalloc_memory()))
tracemalloc.stop()
self.process_snapshot(snapshot, duration)
self.profile_duration += duration
def test_get_tracemalloc_memory(self):
data = [allocate_bytes(123) for count in range(1000)]
size = tracemalloc.get_tracemalloc_memory()
self.assertGreaterEqual(size, 0)
tracemalloc.clear_traces()
size2 = tracemalloc.get_tracemalloc_memory()
self.assertGreaterEqual(size2, 0)
self.assertLessEqual(size2, size)
def test_get_tracemalloc_memory(self):
data = [allocate_bytes(123) for count in range(1000)]
size = tracemalloc.get_tracemalloc_memory()
self.assertGreaterEqual(size, 0)
tracemalloc.clear_traces()
size2 = tracemalloc.get_tracemalloc_memory()
self.assertGreaterEqual(size2, 0)
self.assertLessEqual(size2, size)
def do_kr(x, y, nclusters, verbose, n_init):
start_time = time()
tracemalloc.start()
# Fill in missing values in numeric attributes in advances
xDataFrame = pd.DataFrame(x)
attrList = [0, 3, 4, 5, 6, 8, 9, 11, 12]
numOfRows = x.shape[0]
numOfCols = x.shape[1]
for i in range(0, numOfCols):
if i not in attrList:
colTmp = x[:, i].copy()
colTmp.sort()
if "?" not in colTmp:
continue
missIndex = colTmp.tolist().index("?")
colTmp = list(map(float, colTmp[0:missIndex]))
average = round(mean(colTmp), 2)
for j in range(0, numOfRows):
if xDataFrame.iloc[j, i] == "?":
xDataFrame.iloc[j, i] = average
x = np.asarray(xDataFrame)
kr = kpro.KPrototypes(n_clusters=nclusters,
max_iter=1,
init='random',
n_init=n_init,
verbose=verbose)
kr.fit_predict(x, categorical=attrList)
ari = evaluation.rand(kr.labels_, y)
nmi = evaluation.nmi(kr.labels_, y)
purity = evaluation.purity(kr.labels_, y)
homogenity, completeness, v_measure = homogeneity_completeness_v_measure(
y, kr.labels_)
end_time = time()
elapsedTime = timedelta(seconds=end_time - start_time).total_seconds()
memoryUsage = tracemalloc.get_tracemalloc_memory() / 1024 / 1024
if verbose == 1:
print("Purity = {:8.3f}".format(purity))
print("NMI = {:8.3f}".format(nmi))
print("Homogenity = {:8.3f}".format(homogenity))
print("Completeness = {:8.3f}".format(completeness))
print("V-measure = {:8.3f}".format(v_measure))
print("Elapsed Time = {:8.3f} secs".format(elapsedTime))
print("Memory usage = {:8.3f} MB".format(memoryUsage))
# snapshot = tracemalloc.take_snapshot()
# top_stats = snapshot.statistics('lineno')
# print("[ Top 10 ]")
# for stat in top_stats[:10]:
# print(stat)
tracemalloc.stop()
return [
round(purity, 3),
round(nmi, 3),
round(homogenity, 3),
round(completeness, 3),
round(v_measure, 3),
round(elapsedTime, 3),
round(memoryUsage, 3)
]
def change():
reader = csv.reader(open('french_dictionary.csv', 'r'))
d = {}
for row in reader:
i, j = row
d[i] = j
fin = open("t8.shakespeare.translated.txt", "wt")
with open("t8.shakespeare.txt") as p:
start_time = time.time()
tracemalloc.start()
for line in p:
list2 = []
for words in line.split():
if str("".join(re.findall("[a-zA-Z]+", words))) in d.keys():
list2.append(d[str("".join(re.findall("[a-zA-Z]+",
words)))])
list3[0].append(
str("".join(re.findall("[a-zA-Z]+", words))))
list3[1].append(d[str("".join(
re.findall("[a-zA-Z]+", words)))])
list3[2].append(0)
else:
list2.append(words)
fin.write(" ".join(list2))
print(sorted(list(set(list3[0]))))
print(
"Memory used: ", '{:,.0f}'.format(
tracemalloc.get_tracemalloc_memory() / float(1 << 20)) + " MB")
print("Time to process:", time.time() - start_time, "seconds")
print("Translation done!")
writePerfomance(start_time)
def do_kr(x, y, nclusters, verbose, use_global_attr_count, n_init):
start_time = time()
tracemalloc.start()
categorical = [0, 3, 4, 5, 6, 8, 9, 11, 12]
kr = KCMM(categorical, n_clusters = nclusters, init='random',
n_init = n_init, verbose = verbose, use_global_attr_count = use_global_attr_count)
kr.fit_predict(x)
# print(kr.labels_)
ari = evaluation.rand(kr.labels_, y)
nmi = evaluation.nmi(kr.labels_, y)
purity = evaluation.purity(kr.labels_, y)
homogenity, completeness, v_measure = homogeneity_completeness_v_measure(y, kr.labels_)
end_time = time()
elapsedTime = timedelta(seconds=end_time - start_time).total_seconds()
memoryUsage = tracemalloc.get_tracemalloc_memory() / 1024 / 1024
if verbose == 1:
print("Purity = {:8.3f}" . format(purity))
print("NMI = {:8.3f}" . format(nmi))
print("Homogenity = {:8.3f}" . format(homogenity))
print("Completeness = {:8.3f}" . format(completeness))
print("V-measure = {:8.3f}" . format(v_measure))
print("Elapsed Time = {:8.3f} secs".format(elapsedTime))
print("Memory usage = {:8.3f} MB".format(memoryUsage))
tracemalloc.stop()
return [round(purity,3),round(nmi,3),round(homogenity,3),round(completeness,3),round(v_measure,3),round(elapsedTime,3),round(memoryUsage,3)]
def end(self):
trace_malloc_module_usage = tracemalloc.get_tracemalloc_memory()
print("trace alloc module use memory: %.1f KiB" %
(trace_malloc_module_usage / 1024))
current_size, peak_size = tracemalloc.get_traced_memory()
print("current size: %.1f KiB" % (current_size / 1024))
print("peak size: %.1f KiB" % (peak_size / 1024))
def __exit__(self, exc_type, exc_val, exc_tb):
"""Take a tracemalloc snapshot and subtract the initial snapshot. Also note the tracemalloc memory usage."""
if self.TRACE_ON:
self.tracemalloc_snapshot_finish = tracemalloc.take_snapshot()
self.memory_finish = tracemalloc.get_tracemalloc_memory()
self.statistics = self.tracemalloc_snapshot_finish.compare_to(
self.tracemalloc_snapshot_start, self.statistics_granularity)
self._diff = None
return False
def stop_sampler(self):
logger.debug('Deactivating memory allocation sampler.')
with self.top_lock:
if self.overhead_monitor:
self.overhead_monitor.cancel()
self.overhead_monitor = None
if tracemalloc.is_tracing():
snapshot = tracemalloc.take_snapshot()
logger.debug('Allocation sampler memory overhead %s bytes',
tracemalloc.get_tracemalloc_memory())
tracemalloc.stop()
self.process_snapshot(snapshot)
def add_tracemalloc_metrics(snapshot):
size, max_size = tracemalloc.get_traced_memory()
snapshot.add_metric('tracemalloc.traced.size', size, 'size')
snapshot.add_metric('tracemalloc.traced.max_size', max_size, 'size')
if snapshot.traces:
snapshot.add_metric('tracemalloc.traces', len(snapshot.traces), 'int')
size, free = tracemalloc.get_tracemalloc_memory()
snapshot.add_metric('tracemalloc.module.size', size, 'size')
snapshot.add_metric('tracemalloc.module.free', free, 'size')
if size:
frag = free / size
snapshot.add_metric('tracemalloc.module.fragmentation', frag, 'percent')
def stop_profiler(self):
self.agent.log('Deactivating memory allocation profiler.')
with self.profile_lock:
if self.overhead_monitor:
self.overhead_monitor.cancel()
self.overhead_monitor = None
if tracemalloc.is_tracing():
snapshot = tracemalloc.take_snapshot()
self.agent.log(
'Allocation profiler memory overhead {0} bytes'.format(
tracemalloc.get_tracemalloc_memory()))
tracemalloc.stop()
self.process_snapshot(snapshot, time.time() - self.start_ts)
def _end_trace_malloc(self):
logger.debug(msg="=== START SNAPSHOT ===")
snapshot = tracemalloc.take_snapshot()
snapshot = snapshot.filter_traces(filters=self._get_trace_malloc_filters())
for stat in snapshot.statistics(key_type="lineno", cumulative=True):
logger.debug(msg=f"{stat}")
if self.show_memory:
size, peak = tracemalloc.get_traced_memory()
snapshot_size = tracemalloc.get_tracemalloc_memory()
logger.debug(
msg=f"❕size={self._bytes_to_megabytes(size=size)}, "
f"❗peak={self._bytes_to_megabytes(size=peak)}, "
f"💾snapshot_size={self._bytes_to_megabytes(size=snapshot_size)}"
)
if self.clear_traces:
tracemalloc.clear_traces()
logger.debug(msg="=== END SNAPSHOT ===")
def trace_mem(nframe=6, top=8):
'''
naive memory trace
'''
import tracemalloc
is_tracing = tracemalloc.is_tracing()
if not is_tracing:
# start tracing
tracemalloc.start(nframe)
return {}
else:
# read traced memory alloc
current_mem, peak_mem = tracemalloc.get_traced_memory()
overhead = tracemalloc.get_tracemalloc_memory()
stats = tracemalloc.take_snapshot().statistics('traceback')[:top]
# memory summary
summary = {}
summary['memory'] = int(current_mem // 1024)
summary['peak'] = int(peak_mem // 1024)
summary['overhead'] = int(overhead // 1024)
summary[
'description'] = "traced memory: %d KiB peak: %d KiB overhead: %d KiB" % (
int(current_mem // 1024), int(
peak_mem // 1024), int(overhead // 1024))
# traceback
out_lines = []
for trace in stats:
stacktrace = {}
stacktrace['memory'] = int(trace.size // 1024)
stacktrace['blocks'] = int(trace.count)
stacktrace['stack'] = trace.traceback.format()
out_lines.append(stacktrace)
data = {}
data['summary'] = summary
data['traceback'] = out_lines
# stop tracing
tracemalloc.stop()
return data
def __call__(self, signo, frame):
if not tracemalloc.is_tracing():
print('Starting acquisition...')
tracemalloc.start()
else:
print('Collecting snapshot...')
snapshot = tracemalloc.take_snapshot()
current, peak = tracemalloc.get_traced_memory()
tracer_usage = tracemalloc.get_tracemalloc_memory()
print('Stopping acquisition...')
tracemalloc.stop()
print('---------------------------')
print('Worker %d dumping memtrace' % worker_id)
print('---------------------------')
print('Currently mapped', current, 'bytes; peak was', peak,
'bytes')
print('Tracer used', tracer_usage, 'bytes (before stopping)')
self.print_top(snapshot, limit=self.top_lines)
def bench(func, trace=True, nframe=1):
if trace:
tracemalloc.stop()
tracemalloc.start(nframe)
gc.collect()
best = None
for run in range(BENCH_RUNS):
start = time.monotonic()
func()
dt = time.monotonic() - start
if best is not None:
best = min(best, dt)
else:
best = dt
if trace:
mem = tracemalloc.get_tracemalloc_memory()
ntrace = len(tracemalloc.take_snapshot().traces)
tracemalloc.stop()
else:
mem = ntrace = None
gc.collect()
return best * 1e3, mem, ntrace
def tracemalloc_dump() -> None:
if not tracemalloc.is_tracing():
logger.warning("pid {}: tracemalloc off, nothing to dump"
.format(os.getpid()))
return
# Despite our name for it, `timezone_now` always deals in UTC.
basename = "snap.{}.{}".format(os.getpid(),
timezone_now().strftime("%F-%T"))
path = os.path.join(settings.TRACEMALLOC_DUMP_DIR, basename)
os.makedirs(settings.TRACEMALLOC_DUMP_DIR, exist_ok=True)
gc.collect()
tracemalloc.take_snapshot().dump(path)
procstat = open('/proc/{}/stat'.format(os.getpid()), 'rb').read().split()
rss_pages = int(procstat[23])
logger.info("tracemalloc dump: tracing {} MiB ({} MiB peak), using {} MiB; rss {} MiB; dumped {}"
.format(tracemalloc.get_traced_memory()[0] // 1048576,
tracemalloc.get_traced_memory()[1] // 1048576,
tracemalloc.get_tracemalloc_memory() // 1048576,
rss_pages // 256,
basename))
def tracemalloc_dump() -> None:
if not tracemalloc.is_tracing():
logger.warning("pid {}: tracemalloc off, nothing to dump".format(
os.getpid()))
return
# Despite our name for it, `timezone_now` always deals in UTC.
basename = "snap.{}.{}".format(os.getpid(),
timezone_now().strftime("%F-%T"))
path = os.path.join(settings.TRACEMALLOC_DUMP_DIR, basename)
os.makedirs(settings.TRACEMALLOC_DUMP_DIR, exist_ok=True)
gc.collect()
tracemalloc.take_snapshot().dump(path)
procstat = open('/proc/{}/stat'.format(os.getpid()), 'rb').read().split()
rss_pages = int(procstat[23])
logger.info(
"tracemalloc dump: tracing {} MiB ({} MiB peak), using {} MiB; rss {} MiB; dumped {}"
.format(tracemalloc.get_traced_memory()[0] // 1048576,
tracemalloc.get_traced_memory()[1] // 1048576,
tracemalloc.get_tracemalloc_memory() // 1048576,
rss_pages // 256, basename))
def bench(func, trace=True, nframe=1):
if trace:
tracemalloc.stop()
tracemalloc.start(nframe)
gc.collect()
best = None
for run in range(BENCH_RUNS):
start = time.monotonic()
func()
dt = time.monotonic() - start
if best is not None:
best = min(best, dt)
else:
best = dt
if trace:
mem = tracemalloc.get_tracemalloc_memory()
ntrace = len(tracemalloc.take_snapshot().traces)
tracemalloc.stop()
else:
mem = ntrace = None
gc.collect()
return best * 1e3, mem, ntrace
def main():
args = parse_args()
program_start = time.time()
if args.trace_malloc:
tracemalloc.start()
print(
"Virtual Change/Drift Detection - Association Rule Mining using Python."
)
print("Drift Algorithm: {}".format(args.drift_algorithm))
print("Input file: {}".format(args.input))
print("Output file prefix: {}".format(args.output))
print("Training window size: {}".format(args.training_window_size))
print("Minimum confidence: {}".format(args.min_confidence))
print("Minimum support: {}".format(args.min_support))
print("Minimum lift: {}".format(args.min_lift))
if args.fixed_drift_confidence is not None:
print("Fixed drift confidence of: {}".format(
args.fixed_drift_confidence))
print("Tracing memory allocations: {}".format(args.trace_malloc))
print("Save rules: {}".format(args.save_rules))
print("Generating maximal itemsets: {}".format(args.maximal_itemsets))
reader = iter(DatasetReader(args.input))
transaction_num = 0
end_of_last_window = 0
cohort_num = 1
volatility_detector = make_volatility_detector(args)
while True:
window = take(args.training_window_size, reader)
if len(window) == 0:
break
print("")
print("Mining window [{},{}]".format(transaction_num,
transaction_num + len(window)))
end_of_last_window = transaction_num + len(window)
transaction_num += len(window)
print("Running FP-Growth...", flush=True)
start = time.time()
(itemsets, itemset_counts,
num_transactions) = mine_fp_tree(window, args.min_support,
args.maximal_itemsets)
assert (num_transactions == len(window))
duration = time.time() - start
print("FPGrowth mined {} items in {:.2f} seconds".format(
len(itemsets), duration))
print("Generating rules...", flush=True)
start = time.time()
rules = list(
generate_rules(itemsets, itemset_counts, num_transactions,
args.min_confidence, args.min_lift))
duration = time.time() - start
print("Generated {} rules in {:.2f} seconds".format(
len(rules), duration),
flush=True)
if len(rules) == 0:
print("No rules; just noise. Skipping change detection.")
print(
"Consider increasing training window size or lowering minsup/conf."
)
continue
if args.save_rules:
start = time.time()
output_filename = args.output + "." + str(cohort_num)
cohort_num += 1
write_rules_to_file(rules, output_filename)
print("Wrote rules for cohort {} to file {} in {:.2f} seconds".
format(cohort_num, output_filename, duration),
flush=True)
drift_detector = make_drift_detector(args, volatility_detector)
drift_detector.train(window, rules)
# Read transactions until a drift is detected.
for transaction in reader:
transaction_num += 1
drift = drift_detector.check_for_drift(transaction,
transaction_num)
if drift is not None:
print(
"Detected drift of type {} at transaction {}, {} after end of training window"
.format(drift.drift_type, transaction_num,
transaction_num - end_of_last_window))
if drift.hellinger_value is not None:
print(
"Hellinger value: {}, confidence interval: {} ± {} ([{},{}])"
.format(drift.hellinger_value, drift.mean,
drift.confidence,
drift.mean - drift.confidence,
drift.mean + drift.confidence))
# Record the drift in the volatility detector. This is used inside
# the drift detector to help determine how large a confidence interval
# is required when detecting drifts.
if volatility_detector is not None:
volatility_detector.add(transaction_num)
# Break out of the inner loop, we'll jump back up to the top and mine
# a new training window.
break
if len(window) < args.training_window_size:
break
print("\nEnd of stream\n")
duration = time.time() - program_start
print("Total runtime {:.2f} seconds".format(duration))
if args.trace_malloc:
(_, peak_memory) = tracemalloc.get_traced_memory()
tracemalloc_memory = tracemalloc.get_tracemalloc_memory()
print("Peak memory usage: {:.3f} MB".format(peak_memory / 10**6))
print("tracemalloc overhead: {:.3f} MB".format(
(tracemalloc_memory / 10**6)))
print("Peak memory usage minus tracemalloc overhead: {:.3f} MB".format(
(peak_memory - tracemalloc_memory) / 10**6))
snapshot = tracemalloc.take_snapshot()
bytes_allocated = sum(x.size for x in snapshot.traces)
print("Total traced memory allocated: {:.3f} MB".format(
bytes_allocated / 10**6))
tracemalloc.stop()
return 0
# preform search on state space
res = uniform_cost(subset_dictionary, group_input)
if type(res) == list:
return res[1:]
return res
if __name__ == '__main__':
# f = open("partitionResults.txt", "a+")
group = get_input()
target = sum(group) / 2
print("\nTarget sum is", target)
tracemalloc.start()
start = timeit.default_timer()
uc_res = partition_problem(set(group))
end = timeit.default_timer()
print("result is:", uc_res)
print("sum: ", sum(uc_res))
print("\nRuntime: ", (end - start), "seconds")
print("Memory usage: %d" % tracemalloc.get_tracemalloc_memory(), "Bytes")
# if uc_res != "None":
# f.write("\r\nSet is: {} - length {}\r\nTarget sum: {}\r\n".format(group, len(group), target))
# f.write("\r\r\n==== Uniform Cost =====\r\r\n"
# "Runtime: {} seconds\r\nMemory usage: {} Bytes\r\n"
# "Result set: {}\r\n "
# .format((end-start), tracemalloc.get_tracemalloc_memory(), uc_res))
tracemalloc.stop()
def print_memory_state(tag):
memory = om.convert_units(
tracemalloc.get_traced_memory()[0] -
tracemalloc.get_tracemalloc_memory(), "byte", "Mibyte")
print(f"{tag:50}:", f"{memory:.3f}MiB")
return memory
def writePerfomance(start):
p = open("Performance.txt", "w")
p.write("Time to process:" + str(time.time() - start) + " " + "seconds")
p.write("Memory used: " +
'{:,.0f}'.format(tracemalloc.get_tracemalloc_memory() /
float(1 << 20)) + " MB" + "\n")
def __enter__(self):
"""Take a tracemalloc snapshot."""
if self.TRACE_ON:
self.tracemalloc_snapshot_start = tracemalloc.take_snapshot()
self.memory_start = tracemalloc.get_tracemalloc_memory()
return self
test_names=[]
patterns=[]
plt.rcParams['xtick.labelsize']=7
f1=plt.figure(1)
for algorithm in heuristics:
time_in_s=[]
memory_in_bytes=[]
bm.set_heuristics(algorithm)
text=args.file.read()
text=[line.rstrip("\n\r") for line in text]
text=''.join(text[1:])
pattern=args.pattern
test_name="File: " + str(args.file) + "\nPattern: " + pattern
test_names.append(test_name)
tracemalloc.start()
bm.preprocess(pattern, get_text(text))
mem=round(tracemalloc.get_tracemalloc_memory(), 2)
memory_in_bytes.append(mem)
tracemalloc.stop()
time=min(timeit.repeat('boyer_moore(text)', number=1, repeat=5,
setup='from boyer_moore import boyer_moore; text=\"' + text + "\""))
time_in_s.append(round(time, 2))
print(get_heuristics_name(algorithm) + " finished in " + str(round(time, 2)) + " s and used " + str(mem) + "B")
args.file.close()
def stop_profiler(self):
self.agent.log('Deactivating memory allocation profiler.')
with self.profile_lock:
if self.overhead_monitor:
self.overhead_monitor.cancel()
self.overhead_monitor = None
if tracemalloc.is_tracing():
snapshot = tracemalloc.take_snapshot()
self.agent.log('Allocation profiler memory overhead {0} bytes'.format(tracemalloc.get_tracemalloc_memory()))
tracemalloc.stop()
self.process_snapshot(snapshot, time.time() - self.start_ts)
def update_event(self, inp=-1):
self.set_output_val(0, tracemalloc.get_tracemalloc_memory())
def monitor_overhead():
if tracemalloc.is_tracing() and tracemalloc.get_tracemalloc_memory() > self.MAX_MEMORY_OVERHEAD:
self.agent.log('Allocation profiler memory overhead limit exceeded: {0} bytes'.format(tracemalloc.get_tracemalloc_memory()))
self.stop_profiler()
