def work(self):
"""
Long running loop that periodically dumps the stats.
"""
if profiler.is_running():
raise NovaServiceProfilingException("Profiling already enabled.")
# Set clock type
self.set_clock_type()
# Start profiler
profiler.start()
self._started = utils.utc_seconds()
last_dumped = time.time()
while not self.should_stop():
# Sleep for less than whole interval for faster interrupts
sleep(self._sub_interval)
checked = time.time()
# Only take action if we have exceeded the interval period
if (checked - last_dumped) > self._config.interval:
self._dump()
# Update last dumped
last_dumped = checked
# Finally stop the profiler
profiler.stop()
self._ended = utils.utc_seconds()
def start():
# enable profling by adding to local conf.yaml "with_internal_profiling: True"
# required: "pip install GreenletProfiler"
# Provides function stats in formats 'pstat', 'callgrind', 'ystat'
# stats are saved at "/var/lib/tendrl/profiling/$NS.publisher_id/last_run_func_stat.$stat_type"
# eg: tendrl-node-agent : /var/lib/tendrl/profiling/node_agent/last_run_func_stat.pstat
import atexit
import GreenletProfiler
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
sys.stdout.write("\nStarted Tendrl profiling...")
@atexit.register
def finish():
GreenletProfiler.stop()
sys.stdout.write("\nStopped Tendrl profiling...")
stats = GreenletProfiler.get_func_stats()
_base_path = "/var/lib/tendrl/profiling/{0}/".format(NS.publisher_id)
if not os.path.exists(_base_path):
os.makedirs(_base_path)
for stat_type in ['pstat', 'callgrind', 'ystat']:
_stat_file = "last_run_func_stat.{0}".format(stat_type)
_stat_path = os.path.join(_base_path, _stat_file)
stats.save(_stat_path, type=stat_type)
sys.stdout.write("\nSaved Tendrl profiling stats at %s" % _base_path)
def exit():
print "Entering atexit()"
print 'msgCounter', msgCounter
print 'msgTypeCounter', msgTypeCounter
nums, lens = zip(*msgTypeCounter)
print ' Init Echo Val Aux Coin Ready Share'
print '%8d %8d %9d %9d %9d %9d %9d' % nums[1:]
print '%8d %8d %9d %9d %9d %9d %9d' % lens[1:]
mylog("Total Message size %d" % totalMessageSize, verboseLevel=-2)
if OUTPUT_HALF_MSG:
halfmsgCounter = 0
for msgindex in starting_time.keys():
if msgindex not in ending_time.keys():
logChannel.put(
(msgindex, msgSize[msgindex], msgFrom[msgindex],
msgTo[msgindex], starting_time[msgindex], time.time(),
'[UNRECEIVED]' + repr(msgContent[msgindex])))
halfmsgCounter += 1
mylog('%d extra log exported.' % halfmsgCounter, verboseLevel=-1)
if GEVENT_DEBUG:
checkExceptionPerGreenlet()
if USE_PROFILE:
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
stats.print_all()
stats.save('profile.callgrind', type='callgrind')
def start():
# enable profling by adding to local conf.yaml "with_internal_profiling:
# True"
# required: "pip install GreenletProfiler"
# Provides function stats in formats 'pstat', 'callgrind', 'ystat'
# stats are saved at "/var/lib/tendrl/profiling/$NS.publisher_id
# /last_run_func_stat.$stat_type"
# eg: tendrl-node-agent :
# /var/lib/tendrl/profiling/node_agent/last_run_func_stat.pstat
import atexit
import GreenletProfiler
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
sys.stdout.write("\nStarted Tendrl profiling...")
@atexit.register
def finish():
GreenletProfiler.stop()
sys.stdout.write("\nStopped Tendrl profiling...")
stats = GreenletProfiler.get_func_stats()
_base_path = "/var/lib/tendrl/profiling/{0}/".format(NS.publisher_id)
if not os.path.exists(_base_path):
os.makedirs(_base_path)
for stat_type in ['pstat', 'callgrind', 'ystat']:
_stat_file = "last_run_func_stat.{0}".format(stat_type)
_stat_path = os.path.join(_base_path, _stat_file)
stats.save(_stat_path, type=stat_type)
sys.stdout.write("\nSaved Tendrl profiling stats at %s" % _base_path)
def exit():
print "Entering atexit()"
print 'msgCounter', msgCounter
print 'msgTypeCounter', msgTypeCounter
nums,lens = zip(*msgTypeCounter)
print ' Init Echo Val Aux Coin Ready Share'
print '%8d %8d %9d %9d %9d %9d %9d' % nums[1:]
print '%8d %8d %9d %9d %9d %9d %9d' % lens[1:]
mylog("Total Message size %d" % totalMessageSize, verboseLevel=-2)
if OUTPUT_HALF_MSG:
halfmsgCounter = 0
for msgindex in starting_time.keys():
if msgindex not in ending_time.keys():
logChannel.put((msgindex, msgSize[msgindex], msgFrom[msgindex],
msgTo[msgindex], starting_time[msgindex], time.time(), '[UNRECEIVED]' + repr(msgContent[msgindex])))
halfmsgCounter += 1
mylog('%d extra log exported.' % halfmsgCounter, verboseLevel=-1)
if GEVENT_DEBUG:
checkExceptionPerGreenlet('gevent_debug')
if USE_PROFILE:
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
stats.print_all()
stats.save('profile.callgrind', type='callgrind')
def doCommonProfileStart(self):
self.profile_start = int(time.time())
import GreenletProfiler
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
Game.glog.log2File("doCommonProfile",
"%s|%s\n" % ("start", self.profile_start))
def __init__(self, out_path='.', **kwargs):
if (not os.path.isdir(self.out_path)):
os.makedirs(self.out_path, exist_ok=True)
sydsys().sim.events['run_end'].append(self.coverage_done)
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
def __init__(self, out_path = '.', **kwargs):
if (not os.path.isdir(self.out_path)):
os.makedirs(self.out_path, exist_ok=True)
sydsys().sim.events['run_end'].append(self.coverage_done)
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
def __call__(self, req):
taskid = req.headers['X-Neutron-Profiler-taskid']
action = req.headers['X-Neutron-Profiler-Action']
if action == 'start':
# (anilvenkata): save self.taskid, helpful in throwing error
# if stop called without start
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
LOG.debug("Trace Profiler.start profiling %s ", os.getpid())
elif action == 'stop':
GreenletProfiler.stop()
LOG.debug("Trace Profiler.stop profiling %s ", os.getpid())
stats = GreenletProfiler.get_func_stats()
trace_path = os.path.join(cfg.CONF.trace_profiler.trace_path,
taskid)
utils.ensure_dir(trace_path)
trace_file = os.path.join(trace_path, str(os.getpid()))
LOG.debug("Trace Profiler.writing to trace file %s ", trace_file)
stats.save(trace_file, cfg.CONF.trace_profiler.trace_format)
GreenletProfiler.clear_stats()
else:
LOG.info(
"Invalid profiler action %(action)s with "
" taskid %(taskid)s", {
"action": action,
"taskid": taskid
})
def doCommonProfileEnd(self):
self.profile_end = int(time.time())
import GreenletProfiler
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
stats.save('./callgrind_%s_%s.profile' %
(self.profile_start, self.profile_end),
type='callgrind')
Game.glog.log2File("doCommonProfile",
"%s|%s\n" % ("end", self.profile_end))
def stop(self) -> None:
GreenletProfiler.stop()
# gevent_profiler.detach()
greenlet_file = "{:%Y%m%d_%H%M}_profile_greenlet.callgrind".format(
datetime.now())
greenlet_path = os.path.join(self.datadir, greenlet_file)
stats = GreenletProfiler.get_func_stats()
stats.print_all()
stats.save(greenlet_path, type="callgrind")
def finish():
GreenletProfiler.stop()
sys.stdout.write("\nStopped Tendrl profiling...")
stats = GreenletProfiler.get_func_stats()
_base_path = "/var/lib/tendrl/profiling/{0}/".format(NS.publisher_id)
if not os.path.exists(_base_path):
os.makedirs(_base_path)
for stat_type in ['pstat', 'callgrind', 'ystat']:
_stat_file = "last_run_func_stat.{0}".format(stat_type)
_stat_path = os.path.join(_base_path, _stat_file)
stats.save(_stat_path, type=stat_type)
sys.stdout.write("\nSaved Tendrl profiling stats at %s" % _base_path)
def finish():
GreenletProfiler.stop()
sys.stdout.write("\nStopped Tendrl profiling...")
stats = GreenletProfiler.get_func_stats()
_base_path = "/var/lib/tendrl/profiling/{0}/".format(NS.publisher_id)
if not os.path.exists(_base_path):
os.makedirs(_base_path)
for stat_type in ['pstat', 'callgrind', 'ystat']:
_stat_file = "last_run_func_stat.{0}".format(stat_type)
_stat_path = os.path.join(_base_path, _stat_file)
stats.save(_stat_path, type=stat_type)
sys.stdout.write("\nSaved Tendrl profiling stats at %s" % _base_path)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--transfers', default=100, type=int)
parser.add_argument('--nodes', default=10, type=int)
parser.add_argument('--assets', default=1, type=int)
parser.add_argument('--channels-per-node', default=2, type=int)
parser.add_argument('-p', '--profile', default=False, action='store_true')
args = parser.parse_args()
if args.profile:
import GreenletProfiler
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
# test_mediated_transfer(num_assets=2)
# test_mediated_transfer(num_transfers=1000)
# test_mediated_transfer(num_transfers=1000, num_nodes=10, num_assets=9, channels_per_node=3)
test_mediated_transfer(
num_transfers=args.transfers,
num_nodes=args.nodes,
num_assets=args.assets,
channels_per_node=args.channels_per_node,
)
if args.profile:
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
pstats = GreenletProfiler.convert2pstats(stats)
print_serialization(pstats)
print_slow_path(pstats)
print_slow_function(pstats)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--transfers', default=10000, type=int)
parser.add_argument('--max-locked', default=100, type=int)
parser.add_argument('-p', '--profile', default=False, action='store_true')
args = parser.parse_args()
if args.profile:
import GreenletProfiler
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
transfer_speed(
num_transfers=args.transfers,
max_locked=args.max_locked,
)
if args.profile:
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
pstats = GreenletProfiler.convert2pstats(stats)
print_serialization(pstats)
print_slow_path(pstats)
print_slow_function(pstats)
pstats.sort_stats('time').print_stats()
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--transfers', default=10000, type=int)
parser.add_argument('--max-locked', default=100, type=int)
parser.add_argument('-p', '--profile', default=False, action='store_true')
args = parser.parse_args()
if args.profile:
import GreenletProfiler
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
transfer_speed(
num_transfers=args.transfers,
max_locked=args.max_locked,
)
if args.profile:
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
pstats = GreenletProfiler.convert2pstats(stats)
print_serialization(pstats)
print_slow_path(pstats)
print_slow_function(pstats)
pstats.sort_stats('time').print_stats()
def __call__(self, req):
filename = req.headers['X-Neutron-Profiler-Filename']
action = req.headers['X-Neutron-Profiler-Action']
if action == 'start_trace':
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
else:
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
stats.save(filename, 'pstat')
GreenletProfiler.clear_stats()
def setUpClass(cls):
# Measure the CPU cost of spin() as a baseline.
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
for _ in range(10):
spin(1)
GreenletProfiler.stop()
f_stats = GreenletProfiler.get_func_stats()
spin_stat = find_func(f_stats, 'spin')
GreenletTest.spin_cost = spin_stat.ttot / 10.0
GreenletProfiler.clear_stats()
def setUpClass(cls):
# Measure the CPU cost of spin() as a baseline.
GreenletProfiler.set_clock_type("cpu")
GreenletProfiler.start()
for _ in range(10):
spin(1)
GreenletProfiler.stop()
f_stats = GreenletProfiler.get_func_stats()
spin_stat = find_func(f_stats, "spin")
GreenletTest.spin_cost = spin_stat.ttot / 10.0
GreenletProfiler.clear_stats()
def stop(self):
if not self.profiling:
return
GreenletProfiler.stop()
# gevent_profiler.detach()
greenlet_file = '{:%Y%m%d_%H%M}_profile_greenlet.callgrind'.format(datetime.now())
greenlet_path = os.path.join(self.datadir, greenlet_file)
stats = GreenletProfiler.get_func_stats()
stats.print_all()
stats.save(greenlet_path, type='callgrind')
self.profiling = False
def stop(self):
if not self.profiling:
return
GreenletProfiler.stop()
# gevent_profiler.detach()
greenlet_file = '{:%Y%m%d_%H%M}_profile_greenlet.callgrind'.format(
datetime.now())
greenlet_path = os.path.join(self.datadir, greenlet_file)
stats = GreenletProfiler.get_func_stats()
stats.print_all()
stats.save(greenlet_path, type='callgrind')
self.profiling = False
def __init__(self, datadir: str) -> None:
# create a new file every time instead of overwritting the latest profiling
summary_file = "{:%Y%m%d_%H%M}_profile_summary".format(datetime.now())
stats_file = "{:%Y%m%d_%H%M}_profile_stats".format(datetime.now())
summary_path = os.path.join(datadir, summary_file)
stats_path = os.path.join(datadir, stats_file)
gevent_profiler.set_trace_output(None)
gevent_profiler.set_summary_output(summary_path)
gevent_profiler.set_stats_output(stats_path)
GreenletProfiler.set_clock_type("cpu")
GreenletProfiler.start()
# gevent_profiler.attach()
self.datadir = datadir
def _dump(self):
"""
Dumps the profiling stats.
Also manages clearing stats if clearing each interval.
"""
# If clearing each interval, stop profiler
if self._config.clear_each_interval:
profiler.stop()
self._ended = utils.utc_seconds()
# Dump the stats
stats = profiler.get_func_stats()
ctx = ProfilingContext(
started=self._started, ended=utils.utc_seconds(),
topic=self._topic
)
for o in self._outputs:
try:
o.write(ctx, stats)
except Exception:
# @TODO - Possibly do something with logging
pass
# If clearing each interval, clear stats and restart profiler
if self._config.clear_each_interval:
profiler.clear_stats()
profiler.start()
self.started = utils.utc_seconds()
def start(self):
if self.profiling:
return
# create a new file every time instead of overwritting the latest profiling
summary_file = '{:%Y%m%d_%H%M}_profile_summary'.format(datetime.now())
stats_file = '{:%Y%m%d_%H%M}_profile_stats'.format(datetime.now())
summary_path = os.path.join(self.datadir, summary_file)
stats_path = os.path.join(self.datadir, stats_file)
gevent_profiler.set_trace_output(None)
gevent_profiler.set_summary_output(summary_path)
gevent_profiler.set_stats_output(stats_path)
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
# gevent_profiler.attach()
self.profiling = True
def start(self):
if self.profiling:
return
# create a new file every time instead of overwritting the latest profiling
summary_file = '{:%Y%m%d_%H%M}_profile_summary'.format(datetime.now())
stats_file = '{:%Y%m%d_%H%M}_profile_stats'.format(datetime.now())
summary_path = os.path.join(self.datadir, summary_file)
stats_path = os.path.join(self.datadir, stats_file)
gevent_profiler.set_trace_output(None)
gevent_profiler.set_summary_output(summary_path)
gevent_profiler.set_stats_output(stats_path)
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
# gevent_profiler.attach()
self.profiling = True
def process_request(self, request):
# ignore media
if self._is_path_ignoreable(request, ['/media', '/static']):
return
if not hasattr(request, 'session'):
log.warn('request has no session, this middleware needs to be after the SessionMiddleware')
return
# activate profiling?
if 'greenletprofile' in request.GET:
log.debug("activate profiling")
request.session['greenletprofile'] = True
# stop profiling
if 'greenletprofile-stop' in request.GET and request.session.get('greenletprofile'):
log.debug("deactivate profiling")
request.session['greenletprofile'] = False
if request.session.get('greenletprofile'):
log.debug("start profiling")
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
def func_wrapper(*args, **kwargs):
if GreenletProfiler:
GreenletProfiler.set_clock_type('wall')
GreenletProfiler.start()
ret = func(*args, **kwargs)
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
stats.print_all()
stats.save('profile.callgrind', type='callgrind')
return ret
return func(*args, **kwargs)
def process_response(self, request, response):
if not hasattr(request, 'session'):
return
# are we in a profiler run
if request.session.get('greenletprofile'):
GreenletProfiler.stop()
# store the output
tmpfolder = tempfile.tempdir
tmpfolder = os.path.join(tmpfolder, "profiler")
try:
os.makedirs(os.path.normpath(tmpfolder))
except OSError, e:
if e.errno != errno.EEXIST:
raise
remote_ip = request.META.get('HTTP_X_FORWARDED_FOR', request.META.get('REMOTE_ADDR', 'UNKNOWN'))
log_filename = ("callgrind.%s-%s" % (remote_ip, datetime.now())).replace(" ", "_").replace(":", "-")
location = "%s%s%s" % (tmpfolder, os.sep, log_filename)
stats = GreenletProfiler.get_func_stats()
stats.save(log_filename, type='callgrind')
log.debug("wrote profiling log: %s" % (location))
log.debug("for request path: %s" % (request.META["PATH_INFO"]))
def main():
global alive
st = time.time()
try:
GreenletProfiler.start()
start()
except KeyboardInterrupt:
alive = False
finally:
GreenletProfiler.stop()
et = time.time()
print 'total time:', et - st
print 'CNT:', cnt
print 'QPS:', cnt / (et - st)
print
print
stats = GreenletProfiler.get_func_stats()
stats.print_all()
stats.save('profile.callgrind', type='callgrind')
def test_three_levels(self):
def a():
gr_main.switch()
b()
spin(1)
def b():
spin(5)
gr_main.switch()
c()
def c():
spin(7)
GreenletProfiler.set_clock_type("cpu")
GreenletProfiler.start(builtins=True)
gr_main = greenlet.getcurrent()
g = greenlet.greenlet(a)
g.switch()
spin(2)
g.switch()
spin(3)
g.switch()
self.assertFalse(g, "greenlet not complete")
GreenletProfiler.stop()
ystats = GreenletProfiler.get_func_stats()
# Check the stats for spin().
spin_stat = find_func(ystats, "spin")
self.assertEqual(5, spin_stat.ncall)
self.assertAlmostEqual(18 * self.spin_cost, spin_stat.ttot, places=2, msg="spin()'s total time is wrong")
assert_children(spin_stat, ["range"], "spin() has wrong callees")
# Check the stats for a().
a_stat = find_func(ystats, "a")
self.assertEqual(1, a_stat.ncall, "a() not called once")
assert_children(
a_stat, ["spin", "b", "<method 'switch' of 'greenlet.greenlet' objects>"], "a() has wrong callees"
)
self.assertAlmostEqual(13 * self.spin_cost, a_stat.ttot, places=2, msg="a()'s total time is wrong")
self.assertAlmostEqual(13 * self.spin_cost, a_stat.tavg, places=2, msg="a()'s average time is wrong")
self.assertAlmostEqual(a_stat.tsub, 0, places=2, msg="a()'s subtotal is wrong")
# Check the stats for b().
b_stat = find_func(ystats, "b")
self.assertEqual(1, b_stat.ncall, "b() not called once")
assert_children(
b_stat, ["spin", "c", "<method 'switch' of 'greenlet.greenlet' objects>"], "b() has wrong callees"
)
self.assertAlmostEqual(12 * self.spin_cost, b_stat.ttot, places=2, msg="b()'s total time is wrong")
self.assertAlmostEqual(12 * self.spin_cost, b_stat.tavg, places=2, msg="b()'s average time is wrong")
self.assertAlmostEqual(b_stat.tsub, 0, places=2, msg="b()'s subtotal is wrong")
# Check the stats for c().
c_stat = find_func(ystats, "c")
self.assertEqual(1, c_stat.ncall, "c() not called once")
assert_children(c_stat, ["spin"], "c() has wrong callees")
self.assertAlmostEqual(7 * self.spin_cost, c_stat.ttot, places=2, msg="c()'s total time is wrong")
self.assertAlmostEqual(7 * self.spin_cost, c_stat.tavg, places=2, msg="c()'s average time is wrong")
self.assertAlmostEqual(c_stat.tsub, 0, places=2, msg="c()'s subtotal is wrong")
import GreenletProfiler
from gevent_logic import launch
GreenletProfiler.start()
launch()
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
# stats.print_all()
stats.save('callgrind.greenletprofile', type='callgrind')
def tearDown(self):
GreenletProfiler.stop()
GreenletProfiler.clear_stats()
mylog('%d extra log exported.' % halfmsgCounter, verboseLevel=-1)
if GEVENT_DEBUG:
checkExceptionPerGreenlet('gevent_debug')
if USE_PROFILE:
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
stats.print_all()
stats.save('profile.callgrind', type='callgrind')
if __name__ == '__main__':
# GreenletProfiler.set_clock_type('cpu')
atexit.register(exit)
if USE_PROFILE:
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
from optparse import OptionParser
parser = OptionParser()
parser.add_option("-e", "--ecdsa-keys", dest="ecdsa",
help="Location of ECDSA keys", metavar="KEYS")
parser.add_option("-k", "--threshold-keys", dest="threshold_keys",
help="Location of threshold signature keys", metavar="KEYS")
parser.add_option("-c", "--threshold-enc", dest="threshold_encs",
help="Location of threshold encryption keys", metavar="KEYS")
parser.add_option("-s", "--hosts", dest="hosts",
help="Host list file", metavar="HOSTS", default="~/hosts")
parser.add_option("-n", "--number", dest="n",
help="Number of parties", metavar="N", type="int")
parser.add_option("-p", "--tx-path", dest="txpath",
def coverage_done(self, sim):
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
stats.print_all(filter_in=['*sydpy*'])
stats.save(self.out_path + '/profile.callgrind', type='callgrind')
def __enter__(self):
if self._is_start:
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
return self
def main(default_config_file, is_gateway):
global logger
import futile.logging
logger = futile.logging.get_logger(__name__)
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
config_locations = (".", "/etc/openmtc/gevent", "/etc/openmtc")
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("-f",
"--configfile",
default=None,
help="Location of the configuration file. If "
"unspecified the system will look for a file called %s"
" in these locations: %s" %
(default_config_file, ', '.join(config_locations)))
parser.add_argument("-v",
"--verbose",
action="count",
default=None,
help="Increase verbosity in output. This option can be"
" specified multiple times.")
parser.add_argument("--profiler",
action="store_true",
help="Use GreenletProfiler")
args = parser.parse_args()
configfile = args.configfile
futile.logging.set_default_level(futile.logging.DEBUG)
try:
if not configfile:
import os.path
for d in config_locations:
configfile = os.path.join(os.path.abspath(d),
default_config_file)
logger.debug("Trying config file location: %s", configfile)
if os.path.isfile(configfile):
break
else:
raise ConfigurationError(
"Configuration file %s not found in "
"any of these locations: %s" %
(default_config_file, config_locations))
config = load_config(configfile)
except ConfigurationError as e:
sys.stderr.write(str(e) + "\n")
sys.exit(2)
import openmtc_cse.api
openmtc_cse.api.config = config
import openmtc_server.api
openmtc_server.api.config = config
# TODO: kca:
# Also: set global (non-futile) level?
if "logging" in config: # TODO init logging
# FIXME: This won't work, needs translation to log levels
log_conf = config["logging"]
if args.verbose is None:
futile.logging.set_default_level(
log_conf.get("level") or futile.logging.WARNING)
elif args.verbose >= 2:
futile.logging.set_default_level(futile.logging.DEBUG)
else:
futile.logging.set_default_level(futile.logging.INFO)
logfile = log_conf.get("file")
if logfile:
futile.logging.add_log_file(logfile)
else:
futile.logging.set_default_level(futile.logging.DEBUG)
# make iso8601 logging shut up
logger = futile.logging.get_logger(__name__)
futile.logging.get_logger("iso8601").setLevel(futile.logging.ERROR)
logger.debug("Running OpenMTC")
from itertools import starmap
import signal
from gevent import spawn_later
from gevent.event import Event as GEventEvent
from openmtc_gevent.TaskRunner import GEventTaskRunner
from openmtc_cse.methoddomain import OneM2MMethodDomain
from openmtc_cse.transport import OneM2MTransportDomain
from openmtc_server.platform.default.Event import (ResourceFinishEvent,
NetworkEvent)
from GEventNetworkManager import GEventNetworkManager
from openmtc_server.util.db import load_db_module
omd = OneM2MMethodDomain(config=config)
otd = OneM2MTransportDomain(config=config)
nm = GEventNetworkManager(config=config.get("network_manager", {}))
task_runner = GEventTaskRunner()
_components.append(task_runner)
_timers = set()
db = load_db_module(config)
class Api(object):
PLATFORM = "gevent"
class events(object):
resource_created = ResourceFinishEvent(task_runner.run_task)
resource_deleted = ResourceFinishEvent(task_runner.run_task)
resource_updated = ResourceFinishEvent(task_runner.run_task)
resource_announced = ResourceFinishEvent(task_runner.run_task)
# fired when a network interface appeared
# called with <interface>
interface_created = NetworkEvent(task_runner.run_task)
# fired when a network interface was disappeared
# called with <interface>
interface_removed = NetworkEvent(task_runner.run_task)
# fired when an address appeared on an existing interface
# called with <interface>, <address>
address_created = NetworkEvent(task_runner.run_task)
# fired when an address disappeared on an existing interface
# called with <interface>, <address>
address_removed = NetworkEvent(task_runner.run_task)
start_onem2m_session = db.start_onem2m_session
get_shelve = db.get_shelve
# handle request
handle_onem2m_request = omd.handle_onem2m_request
# send request
send_onem2m_request = otd.send_onem2m_request
send_notify = otd.send_notify
register_point_of_access = otd.register_point_of_access
# connectors and endpoints
register_onem2m_client = otd.register_client
get_onem2m_endpoints = otd.get_endpoints
add_poa_list = otd.add_poa_list
network_manager = nm
run_task = task_runner.run_task
@staticmethod
def set_timer(t, f, *args, **kw):
timer = None
def wrapper():
_timers.discard(timer)
f(*args, **kw)
timer = spawn_later(t, wrapper)
_timers.add(timer)
return timer
@staticmethod
def cancel_timer(timer):
_timers.discard(timer)
timer.kill()
map = map
@staticmethod
def starmap(c, l):
return tuple(starmap(c, l))
Api.db = db
openmtc_cse.api.api = Api
openmtc_cse.api.events = Api.events
openmtc_server.api.api = Api
openmtc_server.api.events = Api.events
shutdown_event = GEventEvent()
gevent.signal(signal.SIGTERM, shutdown_event.set)
gevent.signal(signal.SIGINT, shutdown_event.set)
try:
init_component(otd, Api)
init_component(omd, Api)
init_component(nm, Api)
force = config["database"].get("dropDB")
if force or not db.is_initialized():
db.initialize(force)
omd.init_cse_base()
omd.start()
load_plugins(Api, config.get("plugins", ()), config["global"],
config["onem2m"], is_gateway)
init_plugins()
start_plugins()
logger.info("OpenMTC is running")
except:
logger.exception("Error during startup")
else:
if args.profiler:
import GreenletProfiler
GreenletProfiler.set_clock_type("cpu")
GreenletProfiler.start()
# wait for shutdown event
shutdown_event.wait()
if args.profiler:
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
stats.print_all()
stats.save('profile.callgrind', type='callgrind')
stop_plugins()
stop_components()
for timer in _timers:
try:
timer.kill()
except:
logger.exception("Failed to kill timer %s", timer)
def test_recursion(self):
def r(n):
spin(1)
gr_main.switch()
if n > 1:
r(n - 1)
gr_main.switch()
def s(n):
spin(1)
gr_main.switch()
if n > 1:
s(n - 1)
gr_main.switch()
GreenletProfiler.set_clock_type("cpu")
GreenletProfiler.start(builtins=True)
gr_main = greenlet.getcurrent()
g0 = greenlet.greenlet(partial(r, 10)) # Run r 10 times.
g0.switch()
g1 = greenlet.greenlet(partial(s, 2)) # Run s 2 times.
g1.switch()
greenlets = [g0, g1]
# Run all greenlets to completion.
while greenlets:
runlist = greenlets[:]
for g in runlist:
g.switch()
if not g:
# Finished.
greenlets.remove(g)
GreenletProfiler.stop()
ystats = GreenletProfiler.get_func_stats()
# Check the stats for spin().
spin_stat = find_func(ystats, "spin")
self.assertEqual(12, spin_stat.ncall)
# r() ran spin(1) 10 times, s() ran spin(1) 2 times.
self.assertNear(12, spin_stat.ttot / self.spin_cost)
assert_children(spin_stat, ["range"], "spin() has wrong callees")
# Check the stats for r().
r_stat = find_func(ystats, "r")
self.assertEqual(10, r_stat.ncall)
assert_children(
r_stat, ["spin", "r", "<method 'switch' of 'greenlet.greenlet' objects>"], "r() has wrong callees"
)
self.assertNear(10, r_stat.ttot / self.spin_cost)
self.assertNear(1, r_stat.tavg / self.spin_cost)
self.assertAlmostEqual(0, r_stat.tsub, places=3)
# Check the stats for s().
s_stat = find_func(ystats, "s")
self.assertEqual(2, s_stat.ncall)
assert_children(
s_stat, ["spin", "s", "<method 'switch' of 'greenlet.greenlet' objects>"], "s() has wrong callees"
)
self.assertNear(2, s_stat.ttot / self.spin_cost)
self.assertNear(1, s_stat.tavg / self.spin_cost)
self.assertAlmostEqual(0, s_stat.tsub, places=3)
def tearDown(self):
GreenletProfiler.stop()
GreenletProfiler.clear_stats()
def test_recursion(self):
def r(n):
spin(1)
gr_main.switch()
if n > 1:
r(n - 1)
gr_main.switch()
def s(n):
spin(1)
gr_main.switch()
if n > 1:
s(n - 1)
gr_main.switch()
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start(builtins=True)
gr_main = greenlet.getcurrent()
g0 = greenlet.greenlet(partial(r, 10)) # Run r 10 times.
g0.switch()
g1 = greenlet.greenlet(partial(s, 2)) # Run s 2 times.
g1.switch()
greenlets = [g0, g1]
# Run all greenlets to completion.
while greenlets:
runlist = greenlets[:]
for g in runlist:
g.switch()
if not g:
# Finished.
greenlets.remove(g)
GreenletProfiler.stop()
ystats = GreenletProfiler.get_func_stats()
# Check the stats for spin().
spin_stat = find_func(ystats, 'spin')
self.assertEqual(12, spin_stat.ncall)
# r() ran spin(1) 10 times, s() ran spin(1) 2 times.
self.assertNear(12, spin_stat.ttot / self.spin_cost)
assert_children(spin_stat, ['range'], 'spin() has wrong callees')
# Check the stats for r().
r_stat = find_func(ystats, 'r')
self.assertEqual(10, r_stat.ncall)
assert_children(
r_stat,
['spin', 'r', "<method 'switch' of 'greenlet.greenlet' objects>"],
'r() has wrong callees')
self.assertNear(10, r_stat.ttot / self.spin_cost)
self.assertNear(1, r_stat.tavg / self.spin_cost)
self.assertAlmostEqual(0, r_stat.tsub, places=3)
# Check the stats for s().
s_stat = find_func(ystats, 's')
self.assertEqual(2, s_stat.ncall)
assert_children(
s_stat,
['spin', 's', "<method 'switch' of 'greenlet.greenlet' objects>"],
's() has wrong callees')
self.assertNear(2, s_stat.ttot / self.spin_cost)
self.assertNear(1, s_stat.tavg / self.spin_cost)
self.assertAlmostEqual(0, s_stat.tsub, places=3)
if GEVENT_DEBUG:
checkExceptionPerGreenlet()
if USE_PROFILE:
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
stats.print_all()
stats.save('profile.callgrind', type='callgrind')
if __name__ == '__main__':
# GreenletProfiler.set_clock_type('cpu')
atexit.register(exit)
if USE_PROFILE:
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
from optparse import OptionParser
parser = OptionParser()
parser.add_option("-e",
"--ecdsa-keys",
dest="ecdsa",
help="Location of ECDSA keys",
metavar="KEYS")
parser.add_option("-k",
"--threshold-keys",
dest="threshold_keys",
help="Location of threshold signature keys",
metavar="KEYS")
parser.add_option("-c",
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--transfers', default=100, type=int)
parser.add_argument('--nodes', default=10, type=int)
parser.add_argument('--assets', default=1, type=int)
parser.add_argument('--channels-per-node', default=2, type=int)
parser.add_argument('-p', '--profile', default=False, action='store_true')
parser.add_argument('--pdb', default=False, action='store_true')
parser.add_argument('--throughput', dest='throughput', action='store_true', default=True)
parser.add_argument('--latency', dest='throughput', action='store_false')
parser.add_argument('--log', action='store_true', default=False)
args = parser.parse_args()
if args.log:
slogging.configure(':DEBUG')
if args.profile:
import GreenletProfiler
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
assets = [
sha3('asset:{}'.format(number))[:20]
for number in range(args.assets)
]
amount = 10
apps = setup_apps(
amount,
assets,
args.transfers,
args.nodes,
args.channels_per_node,
)
if args.pdb:
from pyethapp.utils import enable_greenlet_debugger
enable_greenlet_debugger()
try:
if args.throughput:
test_throughput(apps, assets, args.transfers, amount)
else:
test_latency(apps, assets, args.transfers, amount)
except:
import pdb
pdb.xpm()
else:
if args.throughput:
test_throughput(apps, assets, args.transfers, amount)
else:
test_latency(apps, assets, args.transfers, amount)
if args.profile:
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
pstats = GreenletProfiler.convert2pstats(stats)
print_serialization(pstats)
print_slow_path(pstats)
print_slow_function(pstats)
pstats.sort_stats('time').print_stats()
def set_clock_type(self):
"""
Sets the clock type according to config
"""
profiler.set_clock_type(self._config.clock_type)
def coverage_done(self, sim):
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
stats.print_all(filter_in=['*sydpy*'])
stats.save(self.out_path + '/profile.callgrind', type='callgrind')
pstats.strip_dirs().sort_stats('time').print_stats(15)
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--transfers', default=100, type=int)
parser.add_argument('--nodes', default=10, type=int)
parser.add_argument('--assets', default=1, type=int)
parser.add_argument('--channels-per-node', default=2, type=int)
parser.add_argument('-p', '--profile', default=False, action='store_true')
args = parser.parse_args()
if args.profile:
import GreenletProfiler
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
# test_mediated_transfer(num_assets=2)
# test_mediated_transfer(num_transfers=1000)
# test_mediated_transfer(num_transfers=1000, num_nodes=10, num_assets=9, channels_per_node=3)
test_mediated_transfer(
num_transfers=args.transfers,
num_nodes=args.nodes,
num_assets=args.assets,
channels_per_node=args.channels_per_node,
)
if args.profile:
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
def test_three_levels(self):
def a():
gr_main.switch()
b()
spin(1)
def b():
spin(5)
gr_main.switch()
c()
def c():
spin(7)
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start(builtins=True)
gr_main = greenlet.getcurrent()
g = greenlet.greenlet(a)
g.switch()
spin(2)
g.switch()
spin(3)
g.switch()
self.assertFalse(g, 'greenlet not complete')
GreenletProfiler.stop()
ystats = GreenletProfiler.get_func_stats()
# Check the stats for spin().
spin_stat = find_func(ystats, 'spin')
self.assertEqual(5, spin_stat.ncall)
self.assertAlmostEqual(18 * self.spin_cost,
spin_stat.ttot,
places=2,
msg="spin()'s total time is wrong")
assert_children(spin_stat, ['range'], 'spin() has wrong callees')
# Check the stats for a().
a_stat = find_func(ystats, 'a')
self.assertEqual(1, a_stat.ncall, 'a() not called once')
assert_children(
a_stat,
['spin', 'b', "<method 'switch' of 'greenlet.greenlet' objects>"],
'a() has wrong callees')
self.assertAlmostEqual(13 * self.spin_cost,
a_stat.ttot,
places=2,
msg="a()'s total time is wrong")
self.assertAlmostEqual(13 * self.spin_cost,
a_stat.tavg,
places=2,
msg="a()'s average time is wrong")
self.assertAlmostEqual(a_stat.tsub,
0,
places=2,
msg="a()'s subtotal is wrong")
# Check the stats for b().
b_stat = find_func(ystats, 'b')
self.assertEqual(1, b_stat.ncall, 'b() not called once')
assert_children(
b_stat,
['spin', 'c', "<method 'switch' of 'greenlet.greenlet' objects>"],
'b() has wrong callees')
self.assertAlmostEqual(12 * self.spin_cost,
b_stat.ttot,
places=2,
msg="b()'s total time is wrong")
self.assertAlmostEqual(12 * self.spin_cost,
b_stat.tavg,
places=2,
msg="b()'s average time is wrong")
self.assertAlmostEqual(b_stat.tsub,
0,
places=2,
msg="b()'s subtotal is wrong")
# Check the stats for c().
c_stat = find_func(ystats, 'c')
self.assertEqual(1, c_stat.ncall, 'c() not called once')
assert_children(c_stat, ['spin'], 'c() has wrong callees')
self.assertAlmostEqual(7 * self.spin_cost,
c_stat.ttot,
places=2,
msg="c()'s total time is wrong")
self.assertAlmostEqual(7 * self.spin_cost,
c_stat.tavg,
places=2,
msg="c()'s average time is wrong")
self.assertAlmostEqual(c_stat.tsub,
0,
places=2,
msg="c()'s subtotal is wrong")
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--transfers', default=100, type=int)
parser.add_argument('--nodes', default=10, type=int)
parser.add_argument('--tokens', default=1, type=int)
parser.add_argument('--channels-per-node', default=2, type=int)
parser.add_argument('-p', '--profile', default=False, action='store_true')
parser.add_argument('--pdb', default=False, action='store_true')
parser.add_argument('--throughput',
dest='throughput',
action='store_true',
default=True)
parser.add_argument('--latency', dest='throughput', action='store_false')
parser.add_argument('--log', action='store_true', default=False)
args = parser.parse_args()
if args.log:
slogging.configure(':DEBUG')
if args.profile:
import GreenletProfiler
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
tokens = [
sha3('token:{}'.format(number))[:20] for number in range(args.tokens)
]
amount = 10
apps = setup_apps(
amount,
tokens,
args.transfers,
args.nodes,
args.channels_per_node,
)
if args.pdb:
from pyethapp.utils import enable_greenlet_debugger
enable_greenlet_debugger()
try:
if args.throughput:
test_throughput(apps, tokens, args.transfers, amount)
else:
test_latency(apps, tokens, args.transfers, amount)
except:
import pdb
pdb.xpm()
else:
if args.throughput:
test_throughput(apps, tokens, args.transfers, amount)
else:
test_latency(apps, tokens, args.transfers, amount)
if args.profile:
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
pstats = GreenletProfiler.convert2pstats(stats)
print_serialization(pstats)
print_slow_path(pstats)
print_slow_function(pstats)
pstats.sort_stats('time').print_stats()
def __call__(self, req):
taskid = req.headers['X-Neutron-Profiler-taskid']
action = req.headers['X-Neutron-Profiler-Action']
iteration = req.headers.get('X-Neutron-Profiler-Iteration')
profiler_type = req.headers.get('X-Neutron-Profiler-Type')
obj_graph = True if profiler_type == 'objgraph' else False
objcount = (True if
(profiler_type in ['objcount', 'objgraph']) else False)
calltrace = (True if (not profiler_type
or profiler_type == 'calltrace') else False)
trace_path = os.path.join(cfg.CONF.trace_profiler.trace_path, taskid)
ensure_dir(trace_path)
LOG.info(
"Trace Profiler pid %s taskid %s action %s iteration %s"
" profiler_type %s", os.getpid(), taskid, action, iteration,
profiler_type)
if action == 'start':
if calltrace:
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
if objcount and iteration:
objcount_dict[iteration] = len(gc.get_objects())
if obj_graph:
objgraph.get_new_ids()
LOG.info("Trace Profiler.start profiling %s ", os.getpid())
elif action == 'snapshot':
if iteration:
if objcount:
objcount_dict[iteration] = len(gc.get_objects())
if obj_graph:
objgraph_file = os.path.join(
trace_path,
"{}-{}-{}-objgraph.dot".format(socket.gethostname(),
os.getpid(), iteration))
dump_objgrpah(objgraph_file)
elif action == 'stop':
LOG.info("Trace Profiler.stop profiling %s ", os.getpid())
if calltrace:
trace_file = os.path.join(
trace_path, "{}-{}".format(socket.gethostname(),
os.getpid()))
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
LOG.info("Trace Profiler.writing to trace file %s ",
trace_file)
stats.save(trace_file, cfg.CONF.trace_profiler.trace_format)
GreenletProfiler.clear_stats()
if objcount:
objcount_file = os.path.join(
trace_path,
"{}-{}-objcount".format(socket.gethostname(), os.getpid()))
if iteration:
objcount_dict[iteration] = len(gc.get_objects())
with open(objcount_file, 'w') as fp:
json.dump(objcount_dict, fp)
objcount_dict.clear()
if obj_graph:
objgraph_file = os.path.join(
trace_path,
"{}-{}-objgraph.dot".format(socket.gethostname(),
os.getpid()))
dump_objgrpah(objgraph_file)
else:
LOG.warning(
"Invalid profiler action %(action)s with "
" taskid %(taskid)s", {
"action": action,
"taskid": taskid
})
pstats.strip_dirs().sort_stats('time').print_stats(15)
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--transfers', default=100, type=int)
parser.add_argument('--nodes', default=10, type=int)
parser.add_argument('--assets', default=1, type=int)
parser.add_argument('--channels-per-node', default=2, type=int)
parser.add_argument('-p', '--profile', default=False, action='store_true')
args = parser.parse_args()
if args.profile:
import GreenletProfiler
GreenletProfiler.set_clock_type('cpu')
GreenletProfiler.start()
# test_mediated_transfer(num_assets=2)
# test_mediated_transfer(num_transfers=1000)
# test_mediated_transfer(num_transfers=1000, num_nodes=10, num_assets=9, channels_per_node=3)
test_mediated_transfer(
num_transfers=args.transfers,
num_nodes=args.nodes,
num_assets=args.assets,
channels_per_node=args.channels_per_node,
)
if args.profile:
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
def __exit__(self, exc_type, exc_value, traceback):
if self._is_start:
GreenletProfiler.stop()
stats = GreenletProfiler.get_func_stats()
#stats.print_all()
stats.save(self._profile_file, type='pstat')