def test_native_iterparse(self):
"""Displays time and memory used for empty native parsing"""
for _event, _element in ElementTree.iterparse(self.file,
events=('start', 'end')):
# Do something here
pass
print guppy.hpy().heap()
def start(self):
try:
if self.dry_run == False:
# Attempt to connect to the printer
self.do_connect("")
if self.p.printer == None: sys.exit(1)
print "Connecting to printer..."
# Wait for the attempt to succeed or timeout
for x in range(0,50-1):
if self.p.online == True: break
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(0.1)
print ""
if self.p.online == False:
print "Unable to connect to printer: Connection timed-out."
sys.exit(1)
# Wait for the printer to finish connecting and then reset it
time.sleep(2)
self.reset()
# Start the server, display the startup message and start the ioloop
self.server.start()
self.display_startup_message()
self.server.ioloop.start()
except Exception as ex:
print traceback.format_exc()
if args.heaptrace: print hpy().heap()
self.p.disconnect()
exit()
def move_label(dt, data): if dt > 5: print '----- stuttered %d seconds: heap dump -----'%dt print guppy.hpy().heap() data.galaxy_window.window.close() global x_direction, y_direction, x_move, y_move # for this test, the origin is the first named star data.galaxy_window.range_origin_star = data.galaxy_objects.named_stars[0] data.galaxy_window.range_origin_coordinate = data.galaxy_objects.named_stars[0].coordinates gw = data.galaxy_window.window gw._mouse_x += x_move*x_direction gw._mouse_y += y_move*y_direction if (gw._mouse_x > 399): gw._mouse_x = 399 x_direction = x_direction * -1 x_move = random.randint(1,10) elif (gw._mouse_x < 0): gw._mouse_x = 0 x_direction = x_direction * -1 x_move = random.randint(1,10) if (gw._mouse_y > 399): gw._mouse_y = 399 y_direction = y_direction * -1 y_move = random.randint(1,10) elif (gw._mouse_y < 0): gw._mouse_y = 0 y_direction = y_direction * -1 y_move = random.randint(1,10) gw._mouse_in_window = True data.galaxy_window.set_range_info()
def method(self, name, *args):
if guppy:
# We instantiate the heapy environment here
# so that the memory it consumes doesn't hang
# around for the whole process
hpy().heap().stat.dump(self.path)
else:
logger.error("guppy is not availabe, cannot log memory usage!")
def on_key_up(self, key,
scancode = None,
codepoint = None,
modifier = None,
**kwargs):
if key == 104:
import guppy
print guppy.hpy().heap()
def wrapper(*args, **kwargs):
gc.disable()
start = hpy().heap().size
res = func(*args, **kwargs)
end = hpy().heap().size
gc.enable()
mem_log(name=func.__name__, totmembytes=end , memdiffbytes=(end-start), args=self.arg_extractor(*args, **kwargs))
return res
def __citations_from_file(input_file, should_profile):
"""
Generator function that outputs the paper index, and citations for each entry
"""
# Tokens for parsing
index_token = '#index'
citation_token = '#%'
# Predicates for error checking
none_none = lambda *items: all([item is not None for item in items])
all_none = lambda *items: all([item is None for item in items])
# Next entry data
index = None
citations = []
# Global stats
global papers_with_references
total_papers = 0
for line in input_file:
line = line.strip()
# Parse entry, enforcing that data appears in index -> citations order
if line.startswith(index_token):
assert all_none(index) and len(citations) == 0
index = int(line[len(index_token):])
elif line.startswith(citation_token):
assert none_none(index)
new_citation_id = int(line[len(citation_token):])
assert new_citation_id >= 0
citations.append(new_citation_id)
elif len(line) == 0:
total_papers += 1
# Yield this entry if it has any citations,
if none_none(index):
if len(citations) > 0:
papers_with_references += 1
yield index, citations
index = None
citations = []
# Dump the memory usage if we should
if should_profile:
print "\n\nMemory Usage After Parsing Document Citations:"
from guppy import hpy
print hpy().heap()
def show_dict_mem_usage(dict_path=None, verbose=False):
"""
Show dictionary memory usage.
"""
initial_mem = get_mem_usage()
initial_time = time.time()
morph = pymorphy2.MorphAnalyzer(dict_path)
end_time = time.time()
mem_usage = get_mem_usage()
logger.info(
"Memory usage: %0.1fM dictionary, %0.1fM total (load time %0.2fs)",
(mem_usage - initial_mem) / (1024 * 1024),
mem_usage / (1024 * 1024),
end_time - initial_time,
)
if verbose:
try:
from guppy import hpy
hp = hpy()
logger.debug(hp.heap())
except ImportError:
logger.warn("guppy is not installed, detailed info is not available")
def random_sort3(n):
hp = hpy()
print "Heap at the beginning of the functionn", hp.heap()
l = [random.random() for i in range(n)]
l.sort()
print "Heap at the end of the functionn", hp.heap()
return l
def process_response(self, request, response):
if request.GET.has_key('prof'):
h = hpy()
mem_profile = h.heap()
pd = ProfilerData(
view = request.path,
)
self.prof.close()
out = StringIO()
old_stdout = sys.stdout
sys.stdout = out
stats = hotshot.stats.load(self.tmpfile.name)
#stats.strip_dirs()
stats.sort_stats('cumulative')
stats.print_stats()
sys.stdout = old_stdout
stats_str = out.getvalue()
if response and response.content and stats_str:
response.content = "<h1>Instance wide RAM usage</h1><pre>%s</pre><br/><br/><br/><h1>CPU Time for this request</h1><pre>%s</pre>" % (
mem_profile, stats_str
)
pd.profile = "Instance wide RAM usage\n\n%s\n\n\nCPU Time for this request\n\n%s" % (mem_profile, stats_str)
pd.save()
return response
def do_heapy(self):
from guppy import hpy
self.send_response(200)
self.send_header('Content-Type', 'text/plain')
self.send_header('Cache-Control', 'no-cache')
self.end_headers()
self.wfile.write(hpy().heap())
def _init_leak(self):
if not self.leak: return
if self.heap: return
if self.gc: gc.set_debug(gc.DEBUG_LEAK)
if guppy:
self.heap = guppy.hpy()
self.heap.setrelheap()
def on_process_start(self, task):
if not task.manager.options.mem_usage:
return
# start only once
if self.heapy:
return
self.heapy = hpy()
def _init_debug_mode(self):
"""
"""
try:
import resource
import guppy
import signal
print '# DBG MODE: Debug Mode active'
print '# DBG MODE: Initial memory usage : %f (MB)' % (float(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) / 1024 / 1024)
print '# DBG MODE: Collecting initial heap snaphot...'
hp = guppy.hpy()
heap_initial = hp.heap()
heap_initial # make pep8 happy
def handle_signal(signal_number, frame_stack):
self._launch_debug_mode()
signal.signal(signal.SIGHUP, handle_signal)
print '# DBG MODE: Initial heap snaphot collected'
print '# DBG MODE: Do a `kill -HUP %s` to enter the debug mode at anytime' % os.getpid()
print '# DBG MODE: Hitting CTRL+C stop Garuda then enter the debugg mode.'
except:
print '# DBG MODE: Cannot use Debugging Mode. Modules needed: `ipdb`, `resource`, `objgraph` and `guppy`'
self._debug = False
finally:
print ''
def dump_memory_info(stream = None):
close = False
if not stream:
stream = open(_MEMORY_LOG_PATH, 'a')
close = True
if HEAPY:
hp = hpy().heap()
stream.write('heap:\n')
stream.write('{0}\n'.format(hp))
stream.write('heap.byrcs:\n')
stream.write('{0}\n'.format(hp.byrcs))
stream.write('heap[0].byrcs:\n')
stream.write('{0}\n'.format(hp[0].byrcs))
stream.write('heap[0].byid:\n')
stream.write('{0}\n'.format(hp[0].byid))
stream.write('heap[0].byvia:\n')
stream.write('{0}\n'.format(hp[0].byvia))
stream.write('heap[0].byrcs[0].referrers.byrcs:\n')
stream.write('{0}\n'.format(hp[0].byrcs[0].referrers.byrcs))
stream.write('heap[0].byrcs[0].referrers.byrcs[0].referents:\n')
stream.write('{0}\n'.format(hp[0].byrcs[0].referrers.byrcs[0].referents))
stream.write('heap[0].byrcs[0].referrers.byrcs[0].referents.byvia:\n')
stream.write('{0}\n'.format(hp[0].byrcs[0].referrers.byrcs[0].referents.byvia))
else:
stream.write('hpy (heapy) from the guppy package is not available\n')
if close:
stream.close()
def memory_trace_calls(frame, event, arg):
global _LAST_MEMORY_LOG_TIME
t = time.time()
if t - _LAST_MEMORY_LOG_TIME < MEMORY_LOGGING_FREQUENCY:
return
_LAST_MEMORY_LOG_TIME = t
if event != 'call':
return
co = frame.f_code
func_name = co.co_name
if func_name == 'write':
return
func_line_num = frame.f_lineno
file_name = co.co_filename
caller = frame.f_back
caller_line_num = caller.f_lineno
caller_file_name = caller.f_code.co_filename
hp = hpy().heap()
with open(_MEMORY_LOG_PATH, 'a') as out:
out.write('Call to {0}: {1}: {2} (from {3}: {4})\n'.format(
file_name,
func_name,
func_line_num,
caller_file_name,
caller_line_num))
dump_memory_info(out)
out.write('\n')
if func_name in TRACE_LINES_INTO:
memory_trace_lines
return
def test03_joinTable_guppy(self):
import guppy, gc
hpy = guppy.hpy()
def f(ignored, last, first, i):
gc.collect()
next = hpy.heap()
print "SINCE LAST TIME"
print next - last
print "SINCE FOREVER"
print last - first
serial = 0
session = "session" + str(i)
self.server_site.memcache.set(session, str(serial))
headers = {"Cookie": "TWISTED_SESSION=" + session}
d = client.getPage(
"http://127.0.0.1:19481/POKER_REST",
postdata='{"type":"PacketPokerTableJoin","game_id":1}',
headers=headers,
)
d.addCallback(
lambda x: client.getPage(
"http://127.0.0.1:19481/POKER_REST",
postdata='{"type":"PacketPokerTableQuit","game_id":1}',
headers=headers,
)
)
d.addCallback(self.cleanMemcache)
d.addCallback(f, next, first, i + 1)
first = hpy.heap()
i = 1
f(None, first, first, i)
def do_GET(s):
job = re.compile('^/jobs/[0-9]+$')
joblist = re.compile('^/jobs/?$')
if (s.path == '/stats/heap' and 'guppy' in sys.modules):
s.send_response(200)
s.send_header('Content-Type:', 'text/plain')
s.end_headers()
s.wfile.write(hpy().heap())
elif job.match(s.path):
s.send_response(200)
s.send_header('Content-Type:', 'application/json')
s.end_headers()
id = int(s.path.split('/')[2])
result = dict()
result['command'] = s.server.hbq.commands[id].args
result['status'] = s.server.hbq.commands[id].status
result['stdout'] = s.server.hbq.commands[id].stdout
s.wfile.write(json.dumps(result))
elif joblist.match(s.path):
s.send_response(200)
s.send_header('Content-Type:', 'application/json')
s.end_headers()
result = dict()
for i, cmd in enumerate([[x.args, x.status]
for x in s.server.hbq.commands]):
result[i] = dict(command=" ".join(cmd[0]), status=cmd[1])
s.wfile.write(json.dumps(result))
else:
SimpleHTTPRequestHandler.do_GET(s)
def main():
F, J = build_forms()
if 0:
print '\n', '='*50, 'F'
ei = process_form(F)
if 1:
print '\n', '='*50, 'J'
ei = process_form(J)
print '\n', '='*50, 'mem of only eiJ'
del F
del J
printmem()
find_the_memory_thief(ei.cell_integrals()[0].integrand())
#print formatted_analysis(ei, classes=True)
print
print msize
print mtime
try:
from guppy import hpy
hp = hpy()
print "heap:"
print hp.heap()
except:
print "No guppy installed!"
def main(options, args):
my_addr = (options.ip, int(options.port))
logs_path = options.path
print 'Using the following plug-ins:'
print '*', options.routing_m_file
print '*', options.lookup_m_file
print 'Private DHT name:', options.private_dht_name
routing_m_name = '.'.join(os.path.split(options.routing_m_file))[:-3]
routing_m_mod = __import__(routing_m_name, fromlist=[''])
lookup_m_name = '.'.join(os.path.split(options.lookup_m_file))[:-3]
lookup_m_mod = __import__(lookup_m_name, fromlist=[''])
dht = pymdht.Pymdht(my_addr, logs_path,
routing_m_mod,
lookup_m_mod,
options.private_dht_name,
logs_level)
print '\nType "exit" to stop the DHT and exit'
print 'Type "help" if you need'
while (1):
input = sys.stdin.readline().strip().split()
if not input:
continue
command = input[0]
if command == 'help':
print '''
Available commands are:
- help
- fast info_hash bt_port
- exit
- m Memory information
'''
elif command == 'exit':
dht.stop()
break
elif command == 'm':
import guppy
h = guppy.hpy()
print h.heap()
elif command == 'fast':
if len(input) != 3:
print 'usage: fast info_hash bt_port'
continue
try:
info_hash = identifier.Id(input[1])
except (identifier.IdError):
print 'Invalid info_hash (%s)' % input[1]
try:
bt_port = int(input[2])
except:
print 'Invalid bt_port (%r)' % input[2]
continue
success, peers = dht.get_peers(time.time(), info_hash,
_on_peers_found, bt_port)
if not success:
print 'Lookup failed'
if peers:
print '[local] %d peer(s)' % (len(peers))
print peers
def test03_joinTable_guppy(self):
import guppy, gc
hpy = guppy.hpy()
def f(ignored, last, first, i):
gc.collect()
next = hpy.heap()
print 'SINCE LAST TIME'
print next - last
print 'SINCE FOREVER'
print last - first
serial = 0
session = 'session' + str(i)
self.server_site.memcache.set(session, str(serial))
headers = { 'Cookie': 'TWISTED_SESSION='+session }
d = client.getPage("http://127.0.0.1:19481/POKER_REST", postdata = '{"type":"PacketPokerTableJoin","game_id":1}', headers = headers)
d.addCallback(lambda x: client.getPage("http://127.0.0.1:19481/POKER_REST", postdata = '{"type":"PacketPokerTableQuit","game_id":1}', headers = headers))
def cleanMemcache(x):
pokermemcache.memcache_singleton.clear()
pokermemcache.memcache_expiration_singleton.clear()
d.addCallback(cleanMemcache)
d.addCallback(f, next, first, i+1)
first = hpy.heap()
i = 1
f(None, first, first, i)
d = defer.Deferred()
return d
def dumpheap(self):
if hpy is not None:
h = hpy()
h.heap().dump(self.path+"_heap_"+str(self.current_job.replace("/","_"))+".txt", "a")
f = open(self.path+"_heap_"+str(self.current_job).replace("/","_")+"_str.txt","w+")
f.write(str(h.heap()))
f.close()
def dumpMemoryUsage():
'''
This is a function that prints the memory usage of w3af in real time.
@author: Andres Riancho ( [email protected] )
'''
if not DEBUGMEMORY:
pass
else:
hpy = guppy.hpy()
h = hpy.heap()
byrcs = h.byrcs
if isinstance( byrcs, guppy.heapy.UniSet.IdentitySetMulti ):
om.out.debug( str(byrcs) )
for i in xrange(10):
om.out.debug( str(byrcs[i].byvia) )
#om.out.debug( 'The one:' + repr(byrcs[0].byid[0].theone) )
if DEBUGREFERENCES:
for objMemoryUsage in gc.get_objects():
###
### Note: str objects CAN'T be analyzed this way. They can't create loops, so they arent
### handled by the gc ( __cycling__ garcage collector ) .
###
if isinstance( objMemoryUsage, fuzzableRequest.fuzzableRequest ):
om.out.debug('Objects of class fuzzableRequest are referenced by:' )
om.out.debug( str(hpy.iso(objMemoryUsage).sp) )
def get(self):
log = self.application.logger
try:
memstats = {}
import gc
log.info("start GC collect")
memstats["collected"] = gc.collect()
log.info("GC has finished collecting")
try:
import objgraph
memstats["most_common_types"] = dict(objgraph.most_common_types(100))
except ImportError as err:
log.warning("can't create objgraph: %s", err)
try:
from guppy import hpy
hp = hpy()
h = hp.heap()
memstats["heap"] = str(h)
except ImportError as err:
log.warning("can't create heapdump: %s", err)
self.write(memstats)
except Exception as err:
self.write("unable to generate memstats %s" % err)
def getTotals():
"""The transcriptions can be done in chunks to enable parallelism.
This function combines the parts to get the total triphone count, assuming that
transcribed files have been saved as trans001.txt, trans002.txt, etc.
"""
total = {}
files = sorted(os.listdir("."))
trans = []
for i in files:
if i[:5] == "trans" and i[-3:] == "txt":
trans.append(i)
ID = 0
h = hpy()
for i in range(7):
print h.heap()
sents, tot = makeSentenceList(trans[i], ID)
ID = int(sents[-1][0]) + 1
for k, v in tot.items():
if total.has_key(k):
total[k] += v
else:
total[k] = v
del sents
del tot
return total
def _loop(self, args, opts):
if settings.get('MEMDEBUG_WITH_GUPPY', False) and guppy:
heapy = guppy.hpy()
task = Task().next(locked=0, completed=0)
if task:
task.lock()
cmd = ['python', os.path.join(os.getcwd(), 'scrapy-ctl.py'), 'run']
cmd.append('--task-id=%s'%task.id)
if opts.child_logfile:
cmd.append('--logfile=%s'%opts.child_logfile)
cmd.append('--child')
task.start = datetime.now()
process = subprocess.Popen(cmd, shell=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True)
task.result, task.errors = process.communicate()
task.finish = datetime.now()
task.completed = 1
task.save()
timetext.LANG = 'en'
total = task.finish - task.start
log.msg('Finished: %s(%s) in %s'%(task.name, task.id, timetext.stringify(total)), level=log.INFO, domain=task.domain)
if settings.get('MEMDEBUG_WITH_GUPPY', False) and guppy:
log.msg(heapy.heap(), level=log.DEBUG)
heapy.setref()
else:
time.sleep(30)
def main():
rbncls = rbnmol.rbnmol_total_sumzero
if len(sys.argv) >= 2 and sys.argv[1] == "cached":
rbncls = rbnmol_cached.rbnmol_cached_total_sumzero
print "Using cached rbnmol class"
time = 20.0
seeds = 20
copies = 25
rng = random.Random(42)
content = []
for seed in xrange(seeds):
content += [rbncls.generate(10, seed)]*copies
rbnworld = AChemKit.sims_simple.AChemAbstract()
rbnworld.noreactants = 2
rbnworld.react = reaction.react
#events = AChemKit.sims_simple.simulate_itterative_iter(rbnworld, self.content, time, rng)
#events = AChemKit.sims_simple.simulate_stepwise_multiprocessing_iter(rbnworld, self.content, time, rng)
#events = AChemKit.sims_simple.simulate_stepwise_iter(rbnworld, self.content, time, rng)
events = AChemKit.sims_gillespie.simulate_gillespie_iter(rbnworld, content, time, rng)
b = AChemKit.bucket.Bucket(events)
h = hpy()
print h.heap()
def process_request(self, request):
from guppy import hpy
self.usage = 0
self.heapy = hpy()
self.heapy.setrelheap()
def __init__(self, write_csv=False):
super(Heap, self).__init__()
self.heap_urls = self.data['heap_urls'] = {}
self.hp = hpy()
self.write_csv = write_csv
if self.write_csv:
self.csv_writer = csv.writer(open('heap.csv', 'w'))
def do_profiling(X):
n_nghs = 5
debug(X.shape)
lsh.seed()
out = open('dense_mem.txt', 'w')
ct_size = 0
lsh_size = 0
sizes = [1000, 5000, 10000, 20000, 40000, 60000]
h = hpy()
for s in sizes:
h.setrelheap()
root = cover_tree.create(X[:s])
ct_size = h.heap().indisize
debug('querying cover tree')
nghs = cover_tree.knn(n_nghs, X[0], root)
# use the distance from cover tree to initialize lsh
h.setrelheap()
nn = lsh.start(X[:s], nghs[-1].dist)
lsh_size = h.heap().indisize
out.write("%f, %f, %f\n" % (ct_size, lsh_size, s))
del nghs
del root
del nn
out.flush()
out.close()
"""
# update with parameter configuration
for (path, _, _), val in zip(self.pars, config):
code = "sim.%s = val" % (path, )
exec code in {'sim': self.sim, 'val': val}
self.sim.configure()
return self.sim
if __name__ == '__main__':
from guppy import hpy
__h__ = hpy()
from tvb.simulator.backend import driver_conf
driver_conf.using_gpu = using_gpu = 1
from tvb.simulator.lab import *
# this is test driven devlopment speakin how i can help you
model = models.Generic2dOscillator()
conn = connectivity.Connectivity()
conn.speed = array([4.0])
coupling = coupling.Linear(a=0.0152)
hiss = noise.Additive(nsig=ones((2, )) * 2**-10)
heun = integrators.EulerStochastic(dt=2**-4, noise=hiss)
try:
from twisted.conch import manhole, telnet
from twisted.conch.insults import insults
TWISTED_CONCH_AVAILABLE = True
except ImportError:
TWISTED_CONCH_AVAILABLE = False
from scrapy.exceptions import NotConfigured
from scrapy import log, signals
from scrapy.utils.trackref import print_live_refs
from scrapy.utils.engine import print_engine_status
from scrapy.utils.reactor import listen_tcp
try:
import guppy
hpy = guppy.hpy()
except ImportError:
hpy = None
# signal to update telnet variables
# args: telnet_vars
update_telnet_vars = object()
class TelnetConsole(protocol.ServerFactory):
def __init__(self, crawler):
if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'):
raise NotConfigured
if not TWISTED_CONCH_AVAILABLE:
raise NotConfigured
def GET(self):
import guppy
h = guppy.hpy()
return delegate.RawText(str(h.heap()))
def cmd_guppy():
from guppy import hpy
global hp
hp = hpy()
# -*- coding: utf-8 -*-
"""
Created on Mon Feb 05 13:34:00 2018
@author: 李弘一萌
"""
import pandas as pd
from WindPy import w
from guppy import hpy; hp = hpy()
from Performance import Performance_Main
from Signal_Index_Generation import Signal_Index_Generation_Main
from Signal_Generation import Signal_Generation_Main
from Backtest import OI_Strat_Bktest
w.start()
###############################################################################
#设置回测参数
#ITS大于lambda时视为买入信号
para_lambda_optimize = True
para_lambda_best = False
if para_lambda_optimize == True:
para_lambda_list = [x/100.0 for x in range(-100,100)]
else:
if para_lambda_best == False:
para_lambda_list = [0]
else:
def showHeapy():
from guppy import hpy
h = hpy()
print(h.heap())
from guppy import hpy hp = hpy() hp.setrelheap() # do stuff print(hp.heap())
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""Startup script for running CJC directly from the "source" tree."""
#try:
# import psyco
# psyco.profile()
#except ImportError:
# pass
import sys
if len(sys.argv) > 1 and sys.argv[1] == "--memory-profile":
# as soon as possible
from guppy import hpy
heapy = hpy()
sys.argv[1:] = sys.argv[2:]
else:
heapy = None
import os
import glob
base_dir = sys.path[0]
l = glob.glob(os.path.join(base_dir, "../pyxmpp*"))
for p in l:
if os.path.exists(os.path.join(p, "pyxmpp/__init__.py")):
print >> sys.stderr, "PyXMPP sources found in:", p
l = glob.glob(os.path.join(p, "build/lib*"))
if l:
def run(self):
logger.info("Start OIA daemon")
while True:
if log_memory_usage:
try:
logger.info("GC count: %s" % str(gc.get_count()))
h = hpy()
logger.debug(str(h.heap()))
except:
logger.error(traceback.format_exc())
[CurExp, CurFlow] = self.getCurrentRunInformation()
if SystemType == "raptor" and CurExp != "":
time.sleep(10)
continue
all_run_dirs = self.get_run_dirs()
logger.debug('RUNS DIRECTORIES: %s' % all_run_dirs)
# update flow status
logger.debug('runs in p: %s' % self.runs_in_process)
for run in self.runs_in_process:
# check for last flow
logger.debug('runs in p: %s' % run.dat_path)
run.last_flow = run.exp_flows - 1
if run.name == CurExp:
run.last_flow = CurFlow
logger.debug('flows = %d' % run.last_flow)
# check for deleted runs
# TODO
# check for aborted runs
for run in self.runs_in_process:
if run.aborted():
logger.info('run aborted %s' % run.name)
run.killAnalysis()
for block in run.blocks:
if block in self.blocks_to_process:
self.blocks_to_process.remove(block)
self.runs_in_process.remove(run)
self.runs_processed.append(run)
# check for finished runs
#TODO, add timeout per run?
logger.debug('check for finished runs')
for run in self.runs_in_process:
# check whether all blocks have finished
nb_blocks_finished = 0
for block in run.blocks:
if block.status == 'done':
nb_blocks_finished += 1
logger.info("Run %s: %s blocks ready" %
(run.name, nb_blocks_finished))
if nb_blocks_finished == len(run.blocks):
# update status
run.update_status_file()
# write timings
timing_file = os.path.join(run.sigproc_results_path,
'timing.txt')
try:
with open(timing_file, 'a') as f:
f.write(run.gettiming())
except:
logger.error(traceback.format_exc())
#transfer timing.txt
try:
directory_to_transfer = "onboard_results/sigproc_results"
file_to_transfer = "timing.txt"
ret = Transfer(run.name, directory_to_transfer,
file_to_transfer)
if ret == 0:
logger.debug("Transfer registered %s %s %s" %
(run.name, directory_to_transfer,
file_to_transfer))
else:
logger.error(
"Transfer failed %s %s %s, is datacollect running?"
% (run.name, directory_to_transfer,
file_to_transfer))
except:
logger.error(traceback.format_exc())
self.runs_in_process.remove(run)
self.runs_processed.append(run)
else:
logger.debug(run.gettiming())
processed_run_dirs = [run.name for run in self.runs_processed]
logger.debug('RUNS PROCESSED: %s' % processed_run_dirs)
in_process_run_dirs = [run.name for run in self.runs_in_process]
logger.debug('RUNS IN PROCESS: %s' % in_process_run_dirs)
new_run_dirs = list(
set(all_run_dirs) - set(processed_run_dirs) -
set(in_process_run_dirs))
logger.debug('RUNS NEW: %s' % new_run_dirs)
# update /software/config/OIAStatus
self.update_oiastatus_file()
for run in self.runs_in_process:
run.update_status_file()
# add new runs (blocks)
for run_dir in new_run_dirs:
logger.info('NEW RUN DETECTED: %s' % run_dir)
try:
arun = Run(run_dir, config)
logger.info(arun)
except:
logger.error(traceback.format_exc())
continue
if CurExp and not arun.exp_oia_during_run:
continue
if CurExp == arun.name and CurFlow < 1:
continue
if arun.exp_flows < self.flowblocks:
logger.info('skip run: %s, not enough flows' % arun.name)
self.runs_processed.append(arun)
continue
if arun.aborted():
logger.info('skip aborted run: %s' % arun.name)
self.runs_processed.append(arun)
continue
self.runs_in_process.append(arun)
logger.info('arun.exp_oninstranalysis: %s' %
arun.exp_oninstranalysis)
if not arun.exp_oninstranalysis:
continue
# ignore autoanalyze option in explog.txt
logger.info('autoanalyze: %s' % arun.explogdict['autoanalyze'])
#if not arun.explogdict['autoanalyze']: # contains True,False instead of yes, no
# continue
logger.info("ADD %s blocks" % arun.name)
for block in arun.blocks:
if block.status != 'done':
self.blocks_to_process.append(block)
self.printStatus()
# process runs
timestamp = time.time()
logger.info('time since last run check %s' % timestamp)
while self.blocks_to_process:
# wait a while before checking if queue is empty
time.sleep(3)
logger.info(
'Status: Blocks: {0:3d} Beadfind: {1:2d}/{2:2d} Analysis: {3:2d}/{4:2d} Total: {5:2d}/{6:2d}'
.format(
len(self.blocks_to_process),
self.pool.beadfind_counter, self.nb_max_beadfind_jobs,
self.pool.analysis_counter, self.nb_max_analysis_jobs,
self.pool.beadfind_counter +
self.pool.analysis_counter, self.nb_max_jobs))
predicted_total_HOST_memory = 0
predicted_total_GPU_memory = 0
# get list of all different Runs
for run in self.runs_in_process:
blocks_per_run = [
i for i in self.blocks_to_process if i.run == run
]
bf = len([
i for i in self.blocks_to_process
if i.run == run and i.status == 'performJustBeadFind'
])
an = len([
i for i in self.blocks_to_process
if i.run == run and i.status == 'performAnalysis'
])
try:
nb_max_beadfind_jobs = config.getint(
run.exp_chipversion, 'nb_max_beadfind_jobs')
nb_max_analysis_jobs = config.getint(
run.exp_chipversion, 'nb_max_analysis_jobs')
except:
nb_max_beadfind_jobs = config.getint(
'DefaultChip', 'nb_max_beadfind_jobs')
nb_max_analysis_jobs = config.getint(
'DefaultChip', 'nb_max_analysis_jobs')
if len(blocks_per_run):
logger.info(
'Chip: {0:8} Blocks: {1:3d} Beadfind: {2:2d}/{3:2d} Analysis: {4:2d}/{5:2d} ({6})'
.format(run.exp_chipversion, len(blocks_per_run),
bf, nb_max_beadfind_jobs, an,
nb_max_analysis_jobs, run.name))
try:
HOST_memory_requirement_beadfind = config.getint(
run.exp_chipversion,
'HOST_memory_requirement_beadfind')
HOST_memory_requirement_analysis = config.getint(
run.exp_chipversion,
'HOST_memory_requirement_analysis')
GPU_memory_requirement_analysis = config.getint(
run.exp_chipversion,
'GPU_memory_requirement_analysis')
except:
HOST_memory_requirement_beadfind = config.getint(
'DefaultChip', 'HOST_memory_requirement_beadfind')
HOST_memory_requirement_analysis = config.getint(
'DefaultChip', 'HOST_memory_requirement_analysis')
GPU_memory_requirement_analysis = config.getint(
'DefaultChip', 'GPU_memory_requirement_analysis')
predicted_total_HOST_memory += int(
HOST_memory_requirement_beadfind) * bf
predicted_total_HOST_memory += int(
HOST_memory_requirement_analysis) * an
predicted_total_GPU_memory += int(
GPU_memory_requirement_analysis) * an
# every 60 sec
if time.time() - timestamp > 60:
logger.info('HOST: {0} G GPU: {1} G'.format(
predicted_total_HOST_memory / 1073741824,
predicted_total_GPU_memory / 1073741824))
# TODO: run.exp_oia_during_run
# TODO: check for new runs only if no data acquisition
# [CurExp,CurFlow] = self.getCurrentRunInformation()
# every 60 sec check for new run
if time.time() - timestamp > 60:
logger.debug('check for new run')
break
# check status of blocks
for block in self.blocks_to_process:
# check for processed blocks
if block.status == 'processed':
block.status = "idle"
if block.ret == 0:
if block.flow_end == -1:
block.beadfind_done = True
else:
block.successful_processed = block.flow_end + 1
if block.successful_processed == block.flows_total:
block.status = 'sigproc_done'
else:
logger.error(
'Block %s failed with return code %s' %
(block.name, block.ret))
block.nb_attempts += 1
block.sigproc_results_path_tmp = block.sigproc_results_path + "." + str(
block.nb_attempts)
block.beadfind_done = False
block.successful_processed = 0
block.flow_start = -1
block.flow_end = -1
block.status = "idle"
# processed blocks
if block.status == 'sigproc_done' or block.status == 'sigproc_failed':
# 1. rename block / last sigproc attempt
try:
if not os.path.exists(block.sigproc_results_path):
logger.info('rename block %s %s %s' %
(block.name,
block.sigproc_results_path_tmp,
block.sigproc_results_path))
if block.nb_attempts >= config.getint(
'global', 'nb_retries'):
shutil.move(
block.sigproc_results_path + "." +
str(block.nb_attempts - 1),
block.sigproc_results_path)
else:
shutil.move(block.sigproc_results_path_tmp,
block.sigproc_results_path)
except:
logger.error('renaming failed %s' % block.name)
logger.error(traceback.format_exc())
pass
# 2. remove *.step files
if block.status == 'sigproc_done':
try:
for filename in os.listdir(
block.sigproc_results_path):
if fnmatch.fnmatch(filename, 'step.*'):
full_path_to_file = os.path.join(
block.sigproc_results_path,
filename)
logger.info('remove step file: %s' %
full_path_to_file)
os.remove(full_path_to_file)
except:
logger.error('removing step file failed %s' %
block.name)
logger.error(traceback.format_exc())
pass
'''
# 3. generate MD5SUM for each output file
try:
md5sums = {}
for filename in os.listdir(block.sigproc_results_path):
full_filename = os.path.join(block.sigproc_results_path,filename)
if os.path.isdir(full_filename):
continue
with open(full_filename,'rb') as f:
binary_content = f.read()
md5sums[filename] = hashlib.md5(binary_content).hexdigest()
with open(os.path.join(block.sigproc_results_path,'MD5SUMS'), 'w') as f:
for filename,hexdigest in md5sums.items():
f.write("%s %s\n" % (hexdigest,filename))
except:
logger.error(traceback.format_exc())
'''
block.status = 'ready_to_transfer'
if block.status == 'ready_to_transfer':
directory_to_transfer = "onboard_results/sigproc_results/" + "block_" + block.name
file_to_transfer = ""
logger.info("Register transfer: %s %s %s" %
(block.run_name, directory_to_transfer,
file_to_transfer))
ret = Transfer(block.run_name, directory_to_transfer,
file_to_transfer)
if ret == 0:
block.status = "transferred"
try:
open(
os.path.join(block.sigproc_results_path,
'transfer_requested.txt'),
'w').close()
except:
logger.error(traceback.format_exc())
pass
else:
logger.error(
"Transfer failed %s %s %s, is datacollect running?"
% (block.run_name, directory_to_transfer,
file_to_transfer))
if block.status == 'transferred':
logger.debug("DONE: %s" % (block.name))
block.status = 'done'
self.blocks_to_process.remove(block)
try:
ablock = self.get_next_available_job(config)
except:
ablock = None
logger.error(traceback.format_exc())
if ablock:
if ablock.nb_attempts >= config.getint(
'global', 'nb_retries'):
ablock.status = 'sigproc_failed'
else:
ablock.status = 'queued'
ablock.info = '%s-%s' % (ablock.flow_start,
ablock.flow_end)
logger.debug(
'%s submitted (%s-%s)' %
(ablock.name, ablock.flow_start, ablock.flow_end))
self.pool.add_task(ablock)
#wait 10 sec if no blocks are available
time.sleep(10)
#your code goes here
"""
Your program is to use the brute-force approach in order to find the Answer to Life, the Universe, and Everything. More precisely... rewrite small numbers from input to output. Stop processing input after reading in the number 42. All numbers at input are integers of one or two digits.
"""
from guppy import hpy
def solution(l):
for n in l:
if (n == 42):
break
print n
if __name__ == "__main__":
h = hpy()
h.setref()
solution([x for x in xrange(1,100)])
print h.heap()
def __init__(self, dataDir):
dataRepository = datarepo.FileSystemDataRepository(dataDir)
super(HeapProfilerBackend, self).__init__(dataRepository)
self.profiler = guppy.hpy()
def __init(self): super(memoryGetter,self).__init__(): self.memoryData=[] self.endFlag=0 self.h=hpy()
def memory(self, *args):
h = hpy()
self.out(h.heap())
def ccache():
if is_gae:
form = FORM(
P(
TAG.BUTTON(T("Clear CACHE?"),
_type="submit",
_name="yes",
_value="yes")))
else:
cache.ram.initialize()
cache.disk.initialize()
form = FORM(
P(
TAG.BUTTON(T("Clear CACHE?"),
_type="submit",
_name="yes",
_value="yes")),
P(
TAG.BUTTON(T("Clear RAM"),
_type="submit",
_name="ram",
_value="ram")),
P(
TAG.BUTTON(T("Clear DISK"),
_type="submit",
_name="disk",
_value="disk")),
)
if form.accepts(request.vars, session):
session.flash = ""
if is_gae:
if request.vars.yes:
cache.ram.clear()
session.flash += T("Cache Cleared")
else:
clear_ram = False
clear_disk = False
if request.vars.yes:
clear_ram = clear_disk = True
if request.vars.ram:
clear_ram = True
if request.vars.disk:
clear_disk = True
if clear_ram:
cache.ram.clear()
session.flash += T("Ram Cleared")
if clear_disk:
cache.disk.clear()
session.flash += T("Disk Cleared")
redirect(URL(r=request))
try:
from guppy import hpy
hp = hpy()
except ImportError:
hp = False
import shelve
import os
import copy
import time
import math
from gluon import portalocker
ram = {
'entries': 0,
'bytes': 0,
'objects': 0,
'hits': 0,
'misses': 0,
'ratio': 0,
'oldest': time.time(),
'keys': []
}
disk = copy.copy(ram)
total = copy.copy(ram)
disk['keys'] = []
total['keys'] = []
def GetInHMS(seconds):
hours = math.floor(seconds / 3600)
seconds -= hours * 3600
minutes = math.floor(seconds / 60)
seconds -= minutes * 60
seconds = math.floor(seconds)
return (hours, minutes, seconds)
if is_gae:
gae_stats = cache.ram.client.get_stats()
try:
gae_stats['ratio'] = ((gae_stats['hits'] * 100) /
(gae_stats['hits'] + gae_stats['misses']))
except ZeroDivisionError:
gae_stats['ratio'] = T("?")
gae_stats['oldest'] = GetInHMS(time.time() -
gae_stats['oldest_item_age'])
total.update(gae_stats)
else:
# get ram stats directly from the cache object
ram_stats = cache.ram.stats[request.application]
ram['hits'] = ram_stats['hit_total'] - ram_stats['misses']
ram['misses'] = ram_stats['misses']
try:
ram['ratio'] = ram['hits'] * 100 / ram_stats['hit_total']
except (KeyError, ZeroDivisionError):
ram['ratio'] = 0
for key, value in cache.ram.storage.iteritems():
if hp:
ram['bytes'] += hp.iso(value[1]).size
ram['objects'] += hp.iso(value[1]).count
ram['entries'] += 1
if value[0] < ram['oldest']:
ram['oldest'] = value[0]
ram['keys'].append((key, GetInHMS(time.time() - value[0])))
for key in cache.disk.storage:
value = cache.disk.storage[key]
if isinstance(value[1], dict):
disk['hits'] = value[1]['hit_total'] - value[1]['misses']
disk['misses'] = value[1]['misses']
try:
disk['ratio'] = disk['hits'] * 100 / value[1]['hit_total']
except (KeyError, ZeroDivisionError):
disk['ratio'] = 0
else:
if hp:
disk['bytes'] += hp.iso(value[1]).size
disk['objects'] += hp.iso(value[1]).count
disk['entries'] += 1
if value[0] < disk['oldest']:
disk['oldest'] = value[0]
disk['keys'].append((key, GetInHMS(time.time() - value[0])))
ram_keys = ram.keys(
) # ['hits', 'objects', 'ratio', 'entries', 'keys', 'oldest', 'bytes', 'misses']
ram_keys.remove('ratio')
ram_keys.remove('oldest')
for key in ram_keys:
total[key] = ram[key] + disk[key]
try:
total['ratio'] = total['hits'] * 100 / (total['hits'] +
total['misses'])
except (KeyError, ZeroDivisionError):
total['ratio'] = 0
if disk['oldest'] < ram['oldest']:
total['oldest'] = disk['oldest']
else:
total['oldest'] = ram['oldest']
ram['oldest'] = GetInHMS(time.time() - ram['oldest'])
disk['oldest'] = GetInHMS(time.time() - disk['oldest'])
total['oldest'] = GetInHMS(time.time() - total['oldest'])
def key_table(keys):
return TABLE(
TR(TD(B(T('Key'))), TD(B(T('Time in Cache (h:m:s)')))),
*[TR(TD(k[0]), TD('%02d:%02d:%02d' % k[1])) for k in keys],
**dict(_class='cache-keys',
_style="border-collapse: separate; border-spacing: .5em;"))
if not is_gae:
ram['keys'] = key_table(ram['keys'])
disk['keys'] = key_table(disk['keys'])
total['keys'] = key_table(total['keys'])
return dict(form=form,
total=total,
ram=ram,
disk=disk,
object_stats=hp != False)
def actionStats(self):
import gc
import sys
from Ui import UiRequest
from Crypt import CryptConnection
import main
hpy = None
if self.get.get("size") == "1": # Calc obj size
try:
import guppy
hpy = guppy.hpy()
except:
pass
self.sendHeader()
if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
yield "This function is disabled on this proxy"
return
s = time.time()
# Style
yield """
<style>
* { font-family: monospace }
table td, table th { text-align: right; padding: 0px 10px }
.connections td { white-space: nowrap }
.serving-False { opacity: 0.3 }
</style>
"""
# Memory
yield "rev%s | " % config.rev
yield "%s | " % main.file_server.ip_external_list
yield "Port: %s | " % main.file_server.port
yield "IP Network: %s | " % main.file_server.supported_ip_types
yield "Opened: %s | " % main.file_server.port_opened
yield "Crypt: %s, TLSv1.3: %s | " % (CryptConnection.manager.crypt_supported, CryptConnection.ssl.HAS_TLSv1_3)
yield "In: %.2fMB, Out: %.2fMB | " % (
float(main.file_server.bytes_recv) / 1024 / 1024,
float(main.file_server.bytes_sent) / 1024 / 1024
)
yield "Peerid: %s | " % main.file_server.peer_id
yield "Time correction: %.2fs" % main.file_server.getTimecorrection()
try:
import psutil
process = psutil.Process(os.getpid())
mem = process.get_memory_info()[0] / float(2 ** 20)
yield "Mem: %.2fMB | " % mem
yield "Threads: %s | " % len(process.threads())
yield "CPU: usr %.2fs sys %.2fs | " % process.cpu_times()
yield "Files: %s | " % len(process.open_files())
yield "Sockets: %s | " % len(process.connections())
yield "Calc size <a href='?size=1'>on</a> <a href='?size=0'>off</a>"
except Exception:
pass
yield "<br>"
# Connections
yield "<b>Connections</b> (%s, total made: %s, in: %s, out: %s):<br>" % (
len(main.file_server.connections), main.file_server.last_connection_id, main.file_server.num_incoming, main.file_server.num_outgoing
)
yield "<table class='connections'><tr> <th>id</th> <th>type</th> <th>ip</th> <th>open</th> <th>crypt</th> <th>ping</th>"
yield "<th>buff</th> <th>bad</th> <th>idle</th> <th>open</th> <th>delay</th> <th>cpu</th> <th>out</th> <th>in</th> <th>last sent</th>"
yield "<th>wait</th> <th>version</th> <th>time</th> <th>sites</th> </tr>"
for connection in main.file_server.connections:
if "cipher" in dir(connection.sock):
cipher = connection.sock.cipher()[0]
tls_version = connection.sock.version()
else:
cipher = connection.crypt
tls_version = ""
if "time" in connection.handshake and connection.last_ping_delay:
time_correction = connection.handshake["time"] - connection.handshake_time - connection.last_ping_delay
else:
time_correction = 0.0
yield self.formatTableRow([
("%3d", connection.id),
("%s", connection.type),
("%s:%s", (connection.ip, connection.port)),
("%s", connection.handshake.get("port_opened")),
("<span title='%s %s'>%s</span>", (cipher, tls_version, connection.crypt)),
("%6.3f", connection.last_ping_delay),
("%s", connection.incomplete_buff_recv),
("%s", connection.bad_actions),
("since", max(connection.last_send_time, connection.last_recv_time)),
("since", connection.start_time),
("%.3f", max(-1, connection.last_sent_time - connection.last_send_time)),
("%.3f", connection.cpu_time),
("%.0fk", connection.bytes_sent / 1024),
("%.0fk", connection.bytes_recv / 1024),
("<span title='Recv: %s'>%s</span>", (connection.last_cmd_recv, connection.last_cmd_sent)),
("%s", list(connection.waiting_requests.keys())),
("%s r%s", (connection.handshake.get("version"), connection.handshake.get("rev", "?"))),
("%.2fs", time_correction),
("%s", connection.sites)
])
yield "</table>"
# Trackers
yield "<br><br><b>Trackers:</b><br>"
yield "<table class='trackers'><tr> <th>address</th> <th>request</th> <th>successive errors</th> <th>last_request</th></tr>"
from Site import SiteAnnouncer # importing at the top of the file breaks plugins
for tracker_address, tracker_stat in sorted(SiteAnnouncer.global_stats.items()):
yield self.formatTableRow([
("%s", tracker_address),
("%s", tracker_stat["num_request"]),
("%s", tracker_stat["num_error"]),
("%.0f min ago", min(999, (time.time() - tracker_stat["time_request"]) / 60))
])
yield "</table>"
if "AnnounceShare" in PluginManager.plugin_manager.plugin_names:
yield "<br><br><b>Shared trackers:</b><br>"
yield "<table class='trackers'><tr> <th>address</th> <th>added</th> <th>found</th> <th>latency</th> <th>successive errors</th> <th>last_success</th></tr>"
from AnnounceShare import AnnounceSharePlugin
for tracker_address, tracker_stat in sorted(AnnounceSharePlugin.tracker_storage.getTrackers().items()):
yield self.formatTableRow([
("%s", tracker_address),
("%.0f min ago", min(999, (time.time() - tracker_stat["time_added"]) / 60)),
("%.0f min ago", min(999, (time.time() - tracker_stat.get("time_found", 0)) / 60)),
("%.3fs", tracker_stat["latency"]),
("%s", tracker_stat["num_error"]),
("%.0f min ago", min(999, (time.time() - tracker_stat["time_success"]) / 60)),
])
yield "</table>"
# Tor hidden services
yield "<br><br><b>Tor hidden services (status: %s):</b><br>" % main.file_server.tor_manager.status
for site_address, onion in list(main.file_server.tor_manager.site_onions.items()):
yield "- %-34s: %s<br>" % (site_address, onion)
# Db
yield "<br><br><b>Db</b>:<br>"
for db in Db.opened_dbs:
tables = [row["name"] for row in db.execute("SELECT name FROM sqlite_master WHERE type = 'table'").fetchall()]
table_rows = {}
for table in tables:
table_rows[table] = db.execute("SELECT COUNT(*) AS c FROM %s" % table).fetchone()["c"]
db_size = os.path.getsize(db.db_path) / 1024.0 / 1024.0
yield "- %.3fs: %s %.3fMB, table rows: %s<br>" % (
time.time() - db.last_query_time, db.db_path, db_size, json.dumps(table_rows, sort_keys=True)
)
# Sites
yield "<br><br><b>Sites</b>:"
yield "<table>"
yield "<tr><th>address</th> <th>connected</th> <th title='connected/good/total'>peers</th> <th>content.json</th> <th>out</th> <th>in</th> </tr>"
for site in list(self.server.sites.values()):
yield self.formatTableRow([
(
"""<a href='#' onclick='document.getElementById("peers_%s").style.display="initial"; return false'>%s</a>""",
(site.address, site.address)
),
("%s", [peer.connection.id for peer in list(site.peers.values()) if peer.connection and peer.connection.connected]),
("%s/%s/%s", (
len([peer for peer in list(site.peers.values()) if peer.connection and peer.connection.connected]),
len(site.getConnectablePeers(100)),
len(site.peers)
)),
("%s (loaded: %s)", (
len(site.content_manager.contents),
len([key for key, val in dict(site.content_manager.contents).items() if val])
)),
("%.0fk", site.settings.get("bytes_sent", 0) / 1024),
("%.0fk", site.settings.get("bytes_recv", 0) / 1024),
], "serving-%s" % site.settings["serving"])
yield "<tr><td id='peers_%s' style='display: none; white-space: pre' colspan=6>" % site.address
for key, peer in list(site.peers.items()):
if peer.time_found:
time_found = int(time.time() - peer.time_found) / 60
else:
time_found = "--"
if peer.connection:
connection_id = peer.connection.id
else:
connection_id = None
if site.content_manager.has_optional_files:
yield "Optional files: %4s " % len(peer.hashfield)
time_added = (time.time() - peer.time_added) / (60 * 60 * 24)
yield "(#%4s, rep: %2s, err: %s, found: %3s min, add: %.1f day) %30s -<br>" % (connection_id, peer.reputation, peer.connection_error, time_found, time_added, key)
yield "<br></td></tr>"
yield "</table>"
# Big files
yield "<br><br><b>Big files</b>:<br>"
for site in list(self.server.sites.values()):
if not site.settings.get("has_bigfile"):
continue
bigfiles = {}
yield """<a href="#" onclick='document.getElementById("bigfiles_%s").style.display="initial"; return false'>%s</a><br>""" % (site.address, site.address)
for peer in list(site.peers.values()):
if not peer.time_piecefields_updated:
continue
for sha512, piecefield in peer.piecefields.items():
if sha512 not in bigfiles:
bigfiles[sha512] = []
bigfiles[sha512].append(peer)
yield "<div id='bigfiles_%s' style='display: none'>" % site.address
for sha512, peers in bigfiles.items():
yield "<br> - " + sha512 + " (hash id: %s)<br>" % site.content_manager.hashfield.getHashId(sha512)
yield "<table>"
for peer in peers:
yield "<tr><td>" + peer.key + "</td><td>" + peer.piecefields[sha512].tostring() + "</td></tr>"
yield "</table>"
yield "</div>"
# Cmd stats
yield "<div style='float: left'>"
yield "<br><br><b>Sent commands</b>:<br>"
yield "<table>"
for stat_key, stat in sorted(main.file_server.stat_sent.items(), key=lambda i: i[1]["bytes"], reverse=True):
yield "<tr><td>%s</td><td style='white-space: nowrap'>x %s =</td><td>%.0fkB</td></tr>" % (stat_key, stat["num"], stat["bytes"] / 1024)
yield "</table>"
yield "</div>"
yield "<div style='float: left; margin-left: 20%; max-width: 50%'>"
yield "<br><br><b>Received commands</b>:<br>"
yield "<table>"
for stat_key, stat in sorted(main.file_server.stat_recv.items(), key=lambda i: i[1]["bytes"], reverse=True):
yield "<tr><td>%s</td><td style='white-space: nowrap'>x %s =</td><td>%.0fkB</td></tr>" % (stat_key, stat["num"], stat["bytes"] / 1024)
yield "</table>"
yield "</div>"
yield "<div style='clear: both'></div>"
# No more if not in debug mode
if not config.debug:
return
# Object types
obj_count = {}
for obj in gc.get_objects():
obj_type = str(type(obj))
if obj_type not in obj_count:
obj_count[obj_type] = [0, 0]
obj_count[obj_type][0] += 1 # Count
obj_count[obj_type][1] += float(sys.getsizeof(obj)) / 1024 # Size
yield "<br><br><b>Objects in memory (types: %s, total: %s, %.2fkb):</b><br>" % (
len(obj_count),
sum([stat[0] for stat in list(obj_count.values())]),
sum([stat[1] for stat in list(obj_count.values())])
)
for obj, stat in sorted(list(obj_count.items()), key=lambda x: x[1][0], reverse=True): # Sorted by count
yield " - %.1fkb = %s x <a href=\"/Listobj?type=%s\">%s</a><br>" % (stat[1], stat[0], obj, html.escape(obj))
# Classes
class_count = {}
for obj in gc.get_objects():
obj_type = str(type(obj))
if obj_type != "<type 'instance'>":
continue
class_name = obj.__class__.__name__
if class_name not in class_count:
class_count[class_name] = [0, 0]
class_count[class_name][0] += 1 # Count
class_count[class_name][1] += float(sys.getsizeof(obj)) / 1024 # Size
yield "<br><br><b>Classes in memory (types: %s, total: %s, %.2fkb):</b><br>" % (
len(class_count),
sum([stat[0] for stat in list(class_count.values())]),
sum([stat[1] for stat in list(class_count.values())])
)
for obj, stat in sorted(list(class_count.items()), key=lambda x: x[1][0], reverse=True): # Sorted by count
yield " - %.1fkb = %s x <a href=\"/Dumpobj?class=%s\">%s</a><br>" % (stat[1], stat[0], obj, html.escape(obj))
from greenlet import greenlet
objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)]
yield "<br>Greenlets (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
from Worker import Worker
objs = [obj for obj in gc.get_objects() if isinstance(obj, Worker)]
yield "<br>Workers (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
from Connection import Connection
objs = [obj for obj in gc.get_objects() if isinstance(obj, Connection)]
yield "<br>Connections (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
from socket import socket
objs = [obj for obj in gc.get_objects() if isinstance(obj, socket)]
yield "<br>Sockets (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
from msgpack import Unpacker
objs = [obj for obj in gc.get_objects() if isinstance(obj, Unpacker)]
yield "<br>Msgpack unpacker (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
from Site.Site import Site
objs = [obj for obj in gc.get_objects() if isinstance(obj, Site)]
yield "<br>Sites (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
objs = [obj for obj in gc.get_objects() if isinstance(obj, self.server.log.__class__)]
yield "<br>Loggers (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj.name)))
objs = [obj for obj in gc.get_objects() if isinstance(obj, UiRequest)]
yield "<br>UiRequests (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
from Peer import Peer
objs = [obj for obj in gc.get_objects() if isinstance(obj, Peer)]
yield "<br>Peers (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
objs = [(key, val) for key, val in sys.modules.items() if val is not None]
objs.sort()
yield "<br>Modules (%s):<br>" % len(objs)
for module_name, module in objs:
yield " - %.3fkb: %s %s<br>" % (self.getObjSize(module, hpy), module_name, html.escape(repr(module)))
gc.collect() # Implicit grabage collection
yield "Done in %.1f" % (time.time() - s)
return time_taken
times_taken = take_time(created_list)
# plot graph
plt.title("Input Size vs Time taken")
plt.xlabel("Input size")
plt.ylabel("Time taken")
plt.plot(created_list, times_taken)
plt.show()
# ii.Space graph
# creating session context
h = hpy()
# create function that records space used for each input size
def take_space(array):
space_taken = []
for i in range(1, len(array) + 1):
sub_list = array[0:i]
h.setrelheap()
Question3.find_maximum(sub_list)
raw_string = repr(h)
raw_string = raw_string.split()
space = raw_string[10]
space_taken.append(space)
return space_taken
def main():
"""Processes command line to preprocess a file or a directory."""
program_version = "v%s" % __version__
program_shortdesc = __import__('__main__').__doc__.split("\n")[1]
program_license = """%s
Created by Paul Ross on %s.
Copyright 2008-2015. All rights reserved.
Licensed under GPL 2.0
USAGE
""" % (program_shortdesc, str(__date__))
parser = argparse.ArgumentParser(
description=program_license,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
"-c",
action="store_true",
dest="plot_conditional",
default=False,
help=
"Add conditionally included files to the plots. [default: %(default)s]"
)
parser.add_argument("-d",
"--dump",
action="append",
dest="dump",
default=[],
help="""Dump output, additive. Can be:
C - Conditional compilation graph.
F - File names encountered and their count.
I - Include graph.
M - Macro environment.
T - Token count.
R - Macro dependencies as an input to DOT.
[default: %(default)s]""")
parser.add_argument(
"-g",
"--glob",
type=str,
dest="glob",
default="*.*",
help=
"Pattern match to use when processing directories. [default: %(default)s]"
)
parser.add_argument("--heap",
action="store_true",
dest="heap",
default=False,
help="Profile memory usage. [default: %(default)s]")
parser.add_argument(
"-j", "--jobs",
type=int,
dest="jobs",
default=0,
help="""Max simultaneous processes when pre-processing
directories. Zero uses number of native CPUs [%d].
1 means no multiprocessing."""
% multiprocessing.cpu_count() \
+ " [default: %(default)s]"
)
parser.add_argument("-k",
"--keep-going",
action="store_true",
dest="keep_going",
default=False,
help="Keep going. [default: %(default)s]")
parser.add_argument(
"-l", "--loglevel",
type=int,
dest="loglevel",
default=30,
help="Log Level (debug=10, info=20, warning=30, error=40, critical=50)" \
" [default: %(default)s]"
)
parser.add_argument("-o",
"--output",
type=str,
dest="output",
default="out",
help="Output directory. [default: %(default)s]")
parser.add_argument(
"-p",
action="store_true",
dest="ignore_pragma",
default=False,
help="Ignore pragma statements. [default: %(default)s]")
parser.add_argument(
"-r",
"--recursive",
action="store_true",
dest="recursive",
default=False,
help="Recursively process directories. [default: %(default)s]")
parser.add_argument(
"-t",
"--dot",
action="store_true",
dest="include_dot",
default=False,
help="""Write an DOT include dependency table and execute DOT
on it to create a SVG file. [default: %(default)s]""")
parser.add_argument(
"-G",
action="store_true",
dest="gcc_extensions",
default=False,
help=
"""Support GCC extensions. Currently only #include_next. [default: %(default)s]"""
)
parser.add_argument(dest="path", nargs=1, help="Path to source file.")
Cpp.addStandardArguments(parser)
args = parser.parse_args()
# print(' ARGS '.center(75, '-'))
# print(args)
# print(' END: ARGS '.center(75, '-'))
clkStart = time.clock()
# Initialise logging etc.
inPath = args.path[0]
if args.jobs != 1 and os.path.isdir(inPath):
# Multiprocessing
logFormat = '%(asctime)s %(levelname)-8s [%(process)5d] %(message)s'
else:
logFormat = '%(asctime)s %(levelname)-8s %(message)s'
logging.basicConfig(
level=args.loglevel,
format=logFormat,
# datefmt='%y-%m-%d % %H:%M:%S',
stream=sys.stdout)
# Memory usage dump
if args.heap:
try:
from guppy import hpy
except ImportError:
print('Can not profile memory as you do not have guppy installed:' \
' http://guppy-pe.sourceforge.net/')
args.heap = False
# Start memory profiling if requested
if args.heap:
myHeap = hpy()
myHeap.setrelheap()
else:
myHeap = None
# Create objects to pass to pre-processor
myIncH = IncludeHandler.CppIncludeStdOs(
theUsrDirs=args.incUsr or [],
theSysDirs=args.incSys or [],
)
preDefMacros = {}
if args.predefines:
for d in args.predefines:
_tup = d.split('=')
if len(_tup) == 2:
preDefMacros[_tup[0]] = _tup[1] + '\n'
elif len(_tup) == 1:
preDefMacros[_tup[0]] = '\n'
else:
raise ValueError('Can not read macro definition: %s' % d)
# Create the job specification
jobSpec = MainJobSpec(
incHandler=myIncH,
preDefMacros=preDefMacros,
preIncFiles=Cpp.predefinedFileObjects(args),
diagnostic=CppDiagnostic.PreprocessDiagnosticKeepGoing()
if args.keep_going else None,
pragmaHandler=PragmaHandler.PragmaHandlerNull()
if args.ignore_pragma else None,
keepGoing=args.keep_going,
conditionalLevel=2 if args.plot_conditional else 0,
dumpList=args.dump,
helpMap=retOptionMap(parser, args),
includeDOT=args.include_dot,
cmdLine=' '.join(sys.argv),
gccExtensions=args.gcc_extensions,
)
if os.path.isfile(inPath):
preprocessFileToOutput(inPath, args.output, jobSpec)
writeIndexHtml([inPath], args.output, jobSpec)
elif os.path.isdir(inPath):
preprocessDirToOutput(
inPath,
args.output,
jobSpec,
globMatch=args.glob,
recursive=args.recursive,
numJobs=args.jobs,
)
else:
logging.fatal('%s is neither a file or a directory!' % inPath)
return 1
if args.heap and myHeap is not None:
print('Dump of heap:')
h = myHeap.heap()
print(h)
print()
print('Dump of heap byrcs:')
print(h.byrcs)
print()
clkExec = time.clock() - clkStart
print('CPU time = %8.3f (S)' % clkExec)
print('Bye, bye!')
return 0
def cmd_profile(self, conn):
"""Memory profiling for debugging."""
from guppy import hpy
heap = hpy().heap()
print(heap)
# OAuth
OAUTH_AUTHENTICATION = 'liboauth.backend.OAuthBackend' in AUTHENTICATION_BACKENDS
if OAUTH_AUTHENTICATION:
INSTALLED_APPS.append('liboauth')
LOGIN_URL = '/oauth/accounts/login'
SESSION_EXPIRE_AT_BROWSER_CLOSE = True
# URL Redirection white list.
if desktop.conf.REDIRECT_WHITELIST.get():
MIDDLEWARE_CLASSES.append(
'desktop.middleware.EnsureSafeRedirectURLMiddleware')
#Support HTTPS load-balancing
if desktop.conf.SECURE_PROXY_SSL_HEADER.get():
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https')
############################################################
# Necessary for South to not fuzz with tests. Fixed in South 0.7.1
SKIP_SOUTH_TESTS = True
# Set up environment variable so Kerberos libraries look at our private
# ticket cache
os.environ['KRB5CCNAME'] = desktop.conf.KERBEROS.CCACHE_PATH.get()
# Memory
if desktop.conf.MEMORY_PROFILER.get():
MEMORY_PROFILER = hpy()
MEMORY_PROFILER.setrelheap()
def compute_depthmap(arguments):
"""Compute depthmap for a single shot."""
log.setup()
data, udata, neighbors, min_depth, max_depth, shot = arguments
method = udata.config['depthmap_method']
if udata.raw_depthmap_exists(shot.id):
logger.info("Using precomputed raw depthmap {}".format(shot.id))
return
logger.info("Computing depthmap for image {0} with {1}".format(shot.id, method))
hp=hpy()
before=hp.heap()
de = pydense.DepthmapEstimator()
de.set_depth_range(min_depth, max_depth, 100)
de.set_patchmatch_iterations(udata.config['depthmap_patchmatch_iterations'])
de.set_patch_size(udata.config['depthmap_patch_size'])
de.set_min_patch_sd(udata.config['depthmap_min_patch_sd'])
add_views_to_depth_estimator(data,udata, neighbors, de)
if (method == 'BRUTE_FORCE'):
depth, plane, score, nghbr = de.compute_brute_force()
elif (method == 'PATCH_MATCH'):
depth, plane, score, nghbr = de.compute_patch_match()
elif (method == 'PATCH_MATCH_SAMPLE'):
depth, plane, score, nghbr = de.compute_patch_match_sample()
else:
raise ValueError(
'Unknown depthmap method type '
'(must be BRUTE_FORCE, PATCH_MATCH or PATCH_MATCH_SAMPLE)')
good_score = score > udata.config['depthmap_min_correlation_score']
depth = depth * (depth < max_depth) * good_score
# Save and display results
neighbor_ids = [i.id for i in neighbors[1:]]
udata.save_raw_depthmap(shot.id, depth, plane, score, nghbr, neighbor_ids)
depthmap={}
depthmap.update({'depth':depth})
depthmap.update({'plane':plane})
depthmap.update({'score':score})
depthmap.update({'nghbr':nghbr})
depthmap.update({'nghbrs':neighbor_ids})
#raw_depthmap.update({shot.id:depthmap})
data.save_raw_depthmap(shot.id, depthmap)
# after=hp.heap()
# leftover=after-before
# objgraph.show_most_common_types()
# pdb.set_trace()
# after1=hp.heap()
# leftover1=after1-before
# exit()
# print(data)
# print("pid == ",os.getpid())
# print("depthmap== ", hex(id(depthmap)))
# print("raw_depthmap== ", hex(id(raw_depthmap)))
if udata.config['depthmap_save_debug_files']:
image = data.udata_image[shot.id] #load_undistorted_image(shot.id)
#image[:, :, :3] = image[:, :, [2, 1, 0]]
image = scale_down_image(image, depth.shape[1], depth.shape[0])
ply = depthmap_to_ply(shot, depth, image)
# raw_ply.update({shot.id:ply})
# print("ply== ", hex(id(ply)))
# print("raw_ply== ", hex(id(raw_ply)))
with io.open_wt(udata._depthmap_file(shot.id, 'raw.npz.ply')) as fout:
fout.write(ply)
data.save_raw_ply(shot.id,ply)
print('available memory== ', memory_available())
print("current memory usage==", current_memory_usage())
centre_of_mass):
# define output file
out_file_template = '{0}/toy_mc_N_{1}_from_{2}_to_{3}_{4}TeV.root'
output_file_name = out_file_template.format(output_folder, n_toy, start_at,
start_at + n_input_mc,
centre_of_mass)
return output_file_name
if __name__ == "__main__":
if DEBUG:
sys.argv.append("-h")
if TESTRUN:
import doctest
doctest.testmod()
if PROFILE:
import cProfile
import pstats
from guppy import hpy
profile_filename = 'dps.analysis.unfolding_tests.create_toy_mc_from_tree_profile.txt'
cProfile.run('main()', profile_filename)
statsfile = open("profile_stats.txt", "wb")
p = pstats.Stats(profile_filename, stream=statsfile)
stats = p.strip_dirs().sort_stats('cumulative')
stats.print_stats()
h = hpy().heap()
statsfile.write(str(h))
statsfile.close()
sys.exit(0)
sys.exit(main())
def ccache():
form = FORM(
P(TAG.BUTTON("Clear CACHE?", _type="submit", _name="yes", _value="yes")),
P(TAG.BUTTON("Clear RAM", _type="submit", _name="ram", _value="ram")),
P(TAG.BUTTON("Clear DISK", _type="submit", _name="disk", _value="disk")),
)
if form.accepts(request.vars, session):
clear_ram = False
clear_disk = False
session.flash = ""
if request.vars.yes:
clear_ram = clear_disk = True
if request.vars.ram:
clear_ram = True
if request.vars.disk:
clear_disk = True
if clear_ram:
cache.ram.clear()
session.flash += "Ram Cleared "
if clear_disk:
cache.disk.clear()
session.flash += "Disk Cleared"
redirect(URL(r=request, c='appadmin'))
try:
from guppy import hpy; hp=hpy()
except ImportError:
hp = False
import shelve, os, copy, time, math
from gluon import portalocker
ram = {
'bytes': 0,
'objects': 0,
'hits': 0,
'misses': 0,
'ratio': 0,
'oldest': time.time()
}
disk = copy.copy(ram)
total = copy.copy(ram)
for key, value in cache.ram.storage.items():
if isinstance(value, dict):
ram['hits'] = value['hit_total'] - value['misses']
ram['misses'] = value['misses']
try:
ram['ratio'] = ram['hits'] * 100 / value['hit_total']
except (KeyError, ZeroDivisionError):
ram['ratio'] = 0
else:
if hp:
ram['bytes'] += hp.iso(value[1]).size
ram['objects'] += hp.iso(value[1]).count
if value[0] < ram['oldest']:
ram['oldest'] = value[0]
locker = open(os.path.join(request.folder,
'cache/cache.lock'), 'a')
portalocker.lock(locker, portalocker.LOCK_EX)
disk_storage = shelve.open(os.path.join(request.folder, 'cache/cache.shelve'))
try:
for key, value in disk_storage.items():
if isinstance(value, dict):
disk['hits'] = value['hit_total'] - value['misses']
disk['misses'] = value['misses']
try:
disk['ratio'] = disk['hits'] * 100 / value['hit_total']
except (KeyError, ZeroDivisionError):
disk['ratio'] = 0
else:
if hp:
disk['bytes'] += hp.iso(value[1]).size
disk['objects'] += hp.iso(value[1]).count
if value[0] < disk['oldest']:
disk['oldest'] = value[0]
finally:
portalocker.unlock(locker)
locker.close()
disk_storage.close()
total['bytes'] = ram['bytes'] + disk['bytes']
total['objects'] = ram['objects'] + disk['objects']
total['hits'] = ram['hits'] + disk['hits']
total['misses'] = ram['misses'] + disk['misses']
try:
total['ratio'] = total['hits'] * 100 / (total['hits'] + total['misses'])
except (KeyError, ZeroDivisionError):
total['ratio'] = 0
if disk['oldest'] < ram['oldest']:
total['oldest'] = disk['oldest']
else:
total['oldest'] = ram['oldest']
def GetInHMS(seconds):
hours = math.floor(seconds / 3600)
seconds -= hours * 3600
minutes = math.floor(seconds / 60)
seconds -= minutes * 60
seconds = math.floor(seconds)
return (hours, minutes, seconds)
ram['oldest'] = GetInHMS(time.time() - ram['oldest'])
disk['oldest'] = GetInHMS(time.time() - disk['oldest'])
total['oldest'] = GetInHMS(time.time() - total['oldest'])
return dict(form=form, total=total,
ram=ram, disk=disk)
https://gist.github.com/earwig/28a64ffb94d51a608e3d
"""
import ctypes
from ctypes import pythonapi as api
import sys
from types import (BuiltinFunctionType, GetSetDescriptorType, FrameType,
MemberDescriptorType, MethodType)
try:
import guppy
from guppy.heapy import Path
except Exception:
raise ImportError('Cannot use local decorator without guppy!')
hp = guppy.hpy()
def _w(x): # NOSONAR
def f(): # NOSONAR
x # NOSONAR
return f
if sys.version_info < (3, 0):
CellType = type(_w(0).func_closure[0])
else:
CellType = type(_w(0).__closure__[0])
del _w
async def heap(self, ctx):
if ctx.author.id in config.ADMINS:
h = hpy()
val = h.heap()
print(val)
def get_heap_mem_usage(self):
return hpy().heap().size
def actionStats(self):
import gc
import sys
from Ui import UiRequest
from Db import Db
from Crypt import CryptConnection
hpy = None
if self.get.get("size") == "1": # Calc obj size
try:
import guppy
hpy = guppy.hpy()
except:
pass
self.sendHeader()
if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
yield "This function is disabled on this proxy"
raise StopIteration
s = time.time()
main = sys.modules["main"]
# Style
yield """
<style>
* { font-family: monospace }
table td, table th { text-align: right; padding: 0px 10px }
.connections td { white-space: nowrap }
.serving-False { opacity: 0.3 }
</style>
"""
# Memory
try:
yield "rev%s | " % config.rev
yield "%s | " % config.ip_external
yield "Opened: %s | " % main.file_server.port_opened
yield "Crypt: %s | " % CryptConnection.manager.crypt_supported
yield "In: %.2fMB, Out: %.2fMB | " % (
float(main.file_server.bytes_recv) / 1024 / 1024,
float(main.file_server.bytes_sent) / 1024 / 1024)
yield "Peerid: %s | " % main.file_server.peer_id
import psutil
process = psutil.Process(os.getpid())
mem = process.get_memory_info()[0] / float(2**20)
yield "Mem: %.2fMB | " % mem
yield "Threads: %s | " % len(process.threads())
yield "CPU: usr %.2fs sys %.2fs | " % process.cpu_times()
yield "Files: %s | " % len(process.open_files())
yield "Sockets: %s | " % len(process.connections())
yield "Calc size <a href='?size=1'>on</a> <a href='?size=0'>off</a>"
except Exception:
pass
yield "<br>"
# Connections
yield "<b>Connections</b> (%s, total made: %s):<br>" % (len(
main.file_server.connections), main.file_server.last_connection_id)
yield "<table class='connections'><tr> <th>id</th> <th>type</th> <th>ip</th> <th>open</th> <th>crypt</th> <th>ping</th>"
yield "<th>buff</th> <th>bad</th> <th>idle</th> <th>open</th> <th>delay</th> <th>cpu</th> <th>out</th> <th>in</th> <th>last sent</th>"
yield "<th>wait</th> <th>version</th> <th>sites</th> </tr>"
for connection in main.file_server.connections:
if "cipher" in dir(connection.sock):
cipher = connection.sock.cipher()[0]
else:
cipher = connection.crypt
yield self.formatTableRow([
("%3d", connection.id), ("%s", connection.type),
("%s:%s", (connection.ip, connection.port)),
("%s", connection.handshake.get("port_opened")),
("<span title='%s'>%s</span>", (connection.crypt, cipher)),
("%6.3f", connection.last_ping_delay),
("%s", connection.incomplete_buff_recv),
("%s", connection.bad_actions),
("since",
max(connection.last_send_time, connection.last_recv_time)),
("since", connection.start_time),
("%.3f",
connection.last_sent_time - connection.last_send_time),
("%.3f", connection.cpu_time),
("%.0fkB", connection.bytes_sent / 1024),
("%.0fkB", connection.bytes_recv / 1024),
("%s", connection.last_cmd),
("%s", connection.waiting_requests.keys()),
("%s r%s", (connection.handshake.get("version"),
connection.handshake.get("rev", "?"))),
("%s", connection.sites)
])
yield "</table>"
# Tor hidden services
yield "<br><br><b>Tor hidden services (status: %s):</b><br>" % main.file_server.tor_manager.status
for site_address, onion in main.file_server.tor_manager.site_onions.items(
):
yield "- %-34s: %s<br>" % (site_address, onion)
# Db
yield "<br><br><b>Db</b>:<br>"
for db in sys.modules["Db.Db"].opened_dbs:
yield "- %.3fs: %s<br>" % (time.time() - db.last_query_time,
db.db_path.encode("utf8"))
# Sites
yield "<br><br><b>Sites</b>:"
yield "<table>"
yield "<tr><th>address</th> <th>connected</th> <th title='connected/good/total'>peers</th> <th>content.json</th> <th>out</th> <th>in</th> </tr>"
for site in sorted(self.server.sites.values(),
lambda a, b: cmp(a.address, b.address)):
yield self.formatTableRow([
("""<a href='#' onclick='document.getElementById("peers_%s").style.display="initial"; return false'>%s</a>""",
(site.address, site.address)),
("%s", [
peer.connection.id for peer in site.peers.values()
if peer.connection and peer.connection.connected
]),
("%s/%s/%s", (len([
peer for peer in site.peers.values()
if peer.connection and peer.connection.connected
]), len(site.getConnectablePeers(100)), len(site.peers))),
("%s (loaded: %s)",
(len(site.content_manager.contents),
len([
key for key, val in dict(
site.content_manager.contents).iteritems() if val
]))),
("%.0fkB", site.settings.get("bytes_sent", 0) / 1024),
("%.0fkB", site.settings.get("bytes_recv", 0) / 1024),
], "serving-%s" % site.settings["serving"])
yield "<tr><td id='peers_%s' style='display: none; white-space: pre' colspan=6>" % site.address
for key, peer in site.peers.items():
if peer.time_found:
time_found = int(time.time() - peer.time_found) / 60
else:
time_found = "--"
if peer.connection:
connection_id = peer.connection.id
else:
connection_id = None
if site.content_manager.hashfield:
yield "Optional files: %4s " % len(peer.hashfield)
time_added = (time.time() - peer.time_added) / (60 * 60 * 24)
yield "(#%4s, err: %s, found: %3s min, add: %.1f day) %30s -<br>" % (
connection_id, peer.connection_error, time_found,
time_added, key)
yield "<br></td></tr>"
yield "</table>"
# No more if not in debug mode
if not config.debug:
raise StopIteration
# Object types
obj_count = {}
for obj in gc.get_objects():
obj_type = str(type(obj))
if obj_type not in obj_count:
obj_count[obj_type] = [0, 0]
obj_count[obj_type][0] += 1 # Count
obj_count[obj_type][1] += float(sys.getsizeof(obj)) / 1024 # Size
yield "<br><br><b>Objects in memory (types: %s, total: %s, %.2fkb):</b><br>" % (
len(obj_count), sum([stat[0] for stat in obj_count.values()]),
sum([stat[1] for stat in obj_count.values()]))
for obj, stat in sorted(obj_count.items(),
key=lambda x: x[1][0],
reverse=True): # Sorted by count
yield " - %.1fkb = %s x <a href=\"/Listobj?type=%s\">%s</a><br>" % (
stat[1], stat[0], obj, cgi.escape(obj))
# Classes
class_count = {}
for obj in gc.get_objects():
obj_type = str(type(obj))
if obj_type != "<type 'instance'>":
continue
class_name = obj.__class__.__name__
if class_name not in class_count:
class_count[class_name] = [0, 0]
class_count[class_name][0] += 1 # Count
class_count[class_name][1] += float(
sys.getsizeof(obj)) / 1024 # Size
yield "<br><br><b>Classes in memory (types: %s, total: %s, %.2fkb):</b><br>" % (
len(class_count), sum([stat[0] for stat in class_count.values()]),
sum([stat[1] for stat in class_count.values()]))
for obj, stat in sorted(class_count.items(),
key=lambda x: x[1][0],
reverse=True): # Sorted by count
yield " - %.1fkb = %s x <a href=\"/Dumpobj?class=%s\">%s</a><br>" % (
stat[1], stat[0], obj, cgi.escape(obj))
from greenlet import greenlet
objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)]
yield "<br>Greenlets (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(
obj, hpy), cgi.escape(repr(obj).encode("utf8")))
from Worker import Worker
objs = [obj for obj in gc.get_objects() if isinstance(obj, Worker)]
yield "<br>Workers (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(
obj, hpy), cgi.escape(repr(obj)))
from Connection import Connection
objs = [obj for obj in gc.get_objects() if isinstance(obj, Connection)]
yield "<br>Connections (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(
obj, hpy), cgi.escape(repr(obj)))
from socket import socket
objs = [obj for obj in gc.get_objects() if isinstance(obj, socket)]
yield "<br>Sockets (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(
obj, hpy), cgi.escape(repr(obj)))
from msgpack import Unpacker
objs = [obj for obj in gc.get_objects() if isinstance(obj, Unpacker)]
yield "<br>Msgpack unpacker (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(
obj, hpy), cgi.escape(repr(obj)))
from Site import Site
objs = [obj for obj in gc.get_objects() if isinstance(obj, Site)]
yield "<br>Sites (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(
obj, hpy), cgi.escape(repr(obj)))
objs = [
obj for obj in gc.get_objects()
if isinstance(obj, self.server.log.__class__)
]
yield "<br>Loggers (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(
obj, hpy), cgi.escape(repr(obj.name)))
objs = [obj for obj in gc.get_objects() if isinstance(obj, UiRequest)]
yield "<br>UiRequests (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(
obj, hpy), cgi.escape(repr(obj)))
from Peer import Peer
objs = [obj for obj in gc.get_objects() if isinstance(obj, Peer)]
yield "<br>Peers (%s):<br>" % len(objs)
for obj in objs:
yield " - %.1fkb: %s<br>" % (self.getObjSize(
obj, hpy), cgi.escape(repr(obj)))
objs = [(key, val) for key, val in sys.modules.iteritems()
if val is not None]
objs.sort()
yield "<br>Modules (%s):<br>" % len(objs)
for module_name, module in objs:
yield " - %.3fkb: %s %s<br>" % (self.getObjSize(
module, hpy), module_name, cgi.escape(repr(module)))
gc.collect() # Implicit grabage collection
yield "Done in %.1f" % (time.time() - s)
def ccache():
form = FORM(
P(
TAG.BUTTON(T("Clear CACHE?"),
_type="submit",
_name="yes",
_value="yes")),
P(TAG.BUTTON(T("Clear RAM"), _type="submit", _name="ram",
_value="ram")),
P(
TAG.BUTTON(T("Clear DISK"),
_type="submit",
_name="disk",
_value="disk")),
)
if form.accepts(request.vars, session):
clear_ram = False
clear_disk = False
session.flash = ""
if request.vars.yes:
clear_ram = clear_disk = True
if request.vars.ram:
clear_ram = True
if request.vars.disk:
clear_disk = True
if clear_ram:
cache.ram.clear()
session.flash += T("Ram Cleared")
if clear_disk:
cache.disk.clear()
session.flash += T("Disk Cleared")
redirect(URL(r=request))
try:
from guppy import hpy
hp = hpy()
except ImportError:
hp = False
import shelve
import os
import copy
import time
import math
from gluon import portalocker
ram = {
'entries': 0,
'bytes': 0,
'objects': 0,
'hits': 0,
'misses': 0,
'ratio': 0,
'oldest': time.time(),
'keys': []
}
disk = copy.copy(ram)
total = copy.copy(ram)
disk['keys'] = []
total['keys'] = []
def GetInHMS(seconds):
hours = math.floor(seconds / 3600)
seconds -= hours * 3600
minutes = math.floor(seconds / 60)
seconds -= minutes * 60
seconds = math.floor(seconds)
return (hours, minutes, seconds)
for key, value in cache.ram.storage.items():
if isinstance(value, dict):
ram['hits'] = value['hit_total'] - value['misses']
ram['misses'] = value['misses']
try:
ram['ratio'] = ram['hits'] * 100 / value['hit_total']
except (KeyError, ZeroDivisionError):
ram['ratio'] = 0
else:
if hp:
ram['bytes'] += hp.iso(value[1]).size
ram['objects'] += hp.iso(value[1]).count
ram['entries'] += 1
if value[0] < ram['oldest']:
ram['oldest'] = value[0]
ram['keys'].append((key, GetInHMS(time.time() - value[0])))
folder = os.path.join(request.folder, 'cache')
if not os.path.exists(folder):
os.mkdir(folder)
locker = open(os.path.join(folder, 'cache.lock'), 'a')
portalocker.lock(locker, portalocker.LOCK_EX)
disk_storage = shelve.open(os.path.join(folder, 'cache.shelve'))
try:
for key, value in disk_storage.items():
if isinstance(value, dict):
disk['hits'] = value['hit_total'] - value['misses']
disk['misses'] = value['misses']
try:
disk['ratio'] = disk['hits'] * 100 / value['hit_total']
except (KeyError, ZeroDivisionError):
disk['ratio'] = 0
else:
if hp:
disk['bytes'] += hp.iso(value[1]).size
disk['objects'] += hp.iso(value[1]).count
disk['entries'] += 1
if value[0] < disk['oldest']:
disk['oldest'] = value[0]
disk['keys'].append((key, GetInHMS(time.time() - value[0])))
finally:
portalocker.unlock(locker)
locker.close()
disk_storage.close()
total['entries'] = ram['entries'] + disk['entries']
total['bytes'] = ram['bytes'] + disk['bytes']
total['objects'] = ram['objects'] + disk['objects']
total['hits'] = ram['hits'] + disk['hits']
total['misses'] = ram['misses'] + disk['misses']
total['keys'] = ram['keys'] + disk['keys']
try:
total['ratio'] = total['hits'] * 100 / (total['hits'] +
total['misses'])
except (KeyError, ZeroDivisionError):
total['ratio'] = 0
if disk['oldest'] < ram['oldest']:
total['oldest'] = disk['oldest']
else:
total['oldest'] = ram['oldest']
ram['oldest'] = GetInHMS(time.time() - ram['oldest'])
disk['oldest'] = GetInHMS(time.time() - disk['oldest'])
total['oldest'] = GetInHMS(time.time() - total['oldest'])
def key_table(keys):
return TABLE(
TR(TD(B(T('Key'))), TD(B(T('Time in Cache (h:m:s)')))),
*[TR(TD(k[0]), TD('%02d:%02d:%02d' % k[1])) for k in keys],
**dict(_class='cache-keys',
_style="border-collapse: separate; border-spacing: .5em;"))
ram['keys'] = key_table(ram['keys'])
disk['keys'] = key_table(disk['keys'])
total['keys'] = key_table(total['keys'])
return dict(form=form,
total=total,
ram=ram,
disk=disk,
object_stats=hp != False)
return _outer
def decode_profile(profile):
stats = hotshot.stats.load(profile)
#stats.strip_dirs()
stats.sort_stats('time', 'calls')
stats.print_stats(20)
###############################
# memory profiling #
###############################
from guppy import hpy
hpy = hpy()
#SETUP_HPY = False # defined in settings.py
def setup_hpy(reset=False):
print "setting up heapy..."
if (not settings.SETUP_HPY) or (reset):
print "resetting it"
hpy.setrelheap()
settings.SETUP_HPY = True
else:
print "not resetting it"
def profile_memory(log_file):
from twisted.enterprise import adbapi
from twisted.internet import reactor, defer
from twisted.internet.error import CannotListenError
from twisted.internet.protocol import ServerFactory
from twisted.internet.threads import deferToThread
crochet.no_setup()
main_context = context.Context()
all_contexts = [main_context]
session = None
quit_confirm_time = None
try:
from guppy import hpy
heapstats = hpy()
heapstats.setref()
except ImportError:
heapstats = None
class PappySession(object):
"""
An object representing a pappy session. Mainly you'll only use this to get to
the session config.
:ivar config: The configuration settings for the session
:vartype config: :class:`pappyproxy.config.PappyConfig`
"""
def __init__(self, sessconfig):
self.config = sessconfig
except ImportError:
from askbot.models.message import Message as DjangoMessage
from django.utils.translation import ugettext_lazy as _
from askbot.utils.console import ProgressBar
from askbot.utils.slug import slugify
from askbot.models.badges import award_badges_signal, award_badges
from askbot.importers.stackexchange.management import is_ready as importer_is_ready
from optparse import make_option
#from markdown2 import Markdown
#markdowner = Markdown(html4tags=True)
if DEBUGME == True:
from guppy import hpy
from askbot.utils import dummy_transaction as transaction
HEAP = hpy()
else:
#from django.db import transaction
from askbot.utils import dummy_transaction as transaction
xml_read_order = (
'VoteTypes',
'UserTypes',
'Users',
'Users2Votes',
'Badges',
'Users2Badges',
'CloseReasons', #'FlatPages',
'MessageTypes',
'PostHistoryTypes',
'PostTypes',
