def main(): args = vmprof.cli.parse_args(sys.argv[1:]) if args.web: output_mode = OUTPUT_WEB elif args.output: output_mode = OUTPUT_FILE else: output_mode = OUTPUT_CLI if output_mode == OUTPUT_FILE: prof_file = args.output prof_name = prof_file.name else: prof_file = tempfile.NamedTemporaryFile(delete=False) prof_name = prof_file.name vmprof.enable(prof_file.fileno(), args.period, args.mem) try: sys.argv = [args.program] + args.args sys.path.insert(0, os.path.dirname(args.program)) runpy.run_path(args.program, run_name='__main__') except BaseException as e: if not isinstance(e, (KeyboardInterrupt, SystemExit)): raise vmprof.disable() prof_file.close() show_stats(prof_name, output_mode, args) if output_mode != OUTPUT_FILE: os.unlink(prof_name)
def test_basic(): tmpfile = tempfile.NamedTemporaryFile(delete=False) vmprof.enable(tmpfile.fileno()) function_foo() vmprof.disable() tmpfile.close() assert b"function_foo" in gzip.GzipFile(tmpfile.name).read()
def test_is_enabled(self): assert vmprof.is_enabled() == False tmpfile = tempfile.NamedTemporaryFile(delete=False) vmprof.enable(tmpfile.fileno()) assert vmprof.is_enabled() == True vmprof.disable() assert vmprof.is_enabled() == False
def main(): args = vmprof.cli.parse_args(sys.argv[1:]) if args.web: output_mode = OUTPUT_WEB elif args.output: output_mode = OUTPUT_FILE else: output_mode = OUTPUT_CLI if output_mode == OUTPUT_FILE: prof_file = args.output else: prof_file = tempfile.NamedTemporaryFile() vmprof.enable(prof_file.fileno(), args.period) try: sys.argv = [args.program] + args.args runpy.run_path(args.program, run_name='__main__') except BaseException as e: if not isinstance(e, (KeyboardInterrupt, SystemExit)): raise vmprof.disable() show_stats(prof_file.name, output_mode, args)
def finish_profile(): vmprof.disable() self.log.info("Profile created under {filename}", filename=self._profile_filename) # now defer to thread conversion d = deferToThread(convert_profile, self._profile_filename) def on_profile_converted(res): self.log.info("Profile data with {count} log entries generated", count=len(res)) self._finished.callback(res) def on_profile_conversaion_failed(err): self.log.error(err.value) self._finished.errback(err) d.addCallbacks(on_profile_converted, on_profile_conversaion_failed) def cleanup(res): # reset state self._state = Profiler.STATE_STOPPED self._profile_filename = None self._started = None self._finished = None self._profile_id = None d.addBoth(cleanup)
def test_read_bit_by_bit(): tmpfile = tempfile.NamedTemporaryFile(delete=False) vmprof.enable(tmpfile.fileno()) function_foo() vmprof.disable() tmpfile.close() stats = read_profile(tmpfile.name) stats.get_tree()
def test_vmprof_show(): tmpfile = tempfile.NamedTemporaryFile(delete=False) vmprof.enable(tmpfile.fileno()) function_bar() vmprof.disable() tmpfile.close() pp = PrettyPrinter() pp.show(tmpfile.name)
def test_gzip_problem(): tmpfile = tempfile.NamedTemporaryFile(delete=False) vmprof.enable(tmpfile.fileno()) vmprof._gzip_proc.kill() function_foo() with py.test.raises(Exception) as exc_info: vmprof.disable() assert "Error while writing profile" in str(exc_info) tmpfile.close()
def _disable_profiler(self): log.info( "Disable Profiler - wrote to {}".format( PROFILER_FILE_NAME ) ) vmprof.disable() os.close(self._profiler_fd) self._profiler_running = False
def test_read_bit_by_bit(): tmpfile = tempfile.NamedTemporaryFile(delete=False) vmprof.enable(tmpfile.fileno()) function_foo() vmprof.disable() tmpfile.close() with open(tmpfile.name, "rb") as f: period, profiles, virtual_symbols, interp_name = read_prof_bit_by_bit(f) stats = Stats(profiles, virtual_symbols, interp_name) stats.get_tree()
def test_get_profile_path(self): assert vmprof.get_profile_path() == None tmpfile = tempfile.NamedTemporaryFile(delete=False) vmprof.enable(tmpfile.fileno()) if not vmprof.get_profile_path() == tmpfile.name: with open(vmprof.get_profile_path(), 'rb') as fd1: with open(tmpfile.name, "rb") as fd2: assert fd1.read() == fd2.read() vmprof.disable() assert vmprof.get_profile_path() == None
def __call__(self, environ, start_response): prof_file = tempfile.NamedTemporaryFile() vmprof.enable(prof_file.fileno(), 0.0005) self.application(environ, start_response) vmprof.disable() stats = vmprof.read_profile(prof_file.name) stats_log = self.stats(stats, getattr(settings, "VMPROF_EXCLUDE", None)) return HttpResponse("<pre>VMprof \n\n===========\n%s</pre>" % stats_log)
def test_gzip_problem(): tmpfile = tempfile.NamedTemporaryFile(delete=False) vmprof.enable(tmpfile.fileno()) vmprof._gzip_proc.kill() vmprof._gzip_proc.wait() # ensure that the gzip process really tries to write # to the gzip proc that was killed function_foo() with py.test.raises(Exception) as exc_info: vmprof.disable() assert "Error while writing profile" in str(exc_info) tmpfile.close()
def test_basic(): tmpfile = tempfile.NamedTemporaryFile(delete=False) vmprof.enable(tmpfile.fileno()) function_foo() vmprof.disable() tmpfile.close() if GZIP: assert b"function_foo" in gzip.GzipFile(tmpfile.name).read() else: with open(tmpfile.name, 'rb') as file: content = file.read() assert b"function_foo" in content
def test_line_profiling(): tmpfile = tempfile.NamedTemporaryFile(delete=False) vmprof.enable(tmpfile.fileno(), lines=True, native=False) # enable lines profiling function_foo() vmprof.disable() tmpfile.close() def walk(tree): assert len(tree.lines) >= len(tree.children) for v in six.itervalues(tree.children): walk(v) stats = read_profile(tmpfile.name) walk(stats.get_tree())
def vmprof_signal_handler(signal, frame): import vmprof curr_proc = multiprocessing.current_process() logger = logging.getLogger('eyrie.script.profile') if getattr(curr_proc, 'vmprof_enabled', False): logger.warn('Disabling vmprof, output path: %s', curr_proc.profile_output_path) vmprof.disable() curr_proc.vmprof_enabled = False else: fileno, output_path = mkstemp(dir=curr_proc.profile_output_dir) curr_proc.profile_output_path = output_path logger.warn('Enabling vmprof, output path: %s', output_path) vmprof.enable(fileno) curr_proc.vmprof_enabled = True
def test_line_profiling(): tmpfile = tempfile.NamedTemporaryFile(delete=False) vmprof.enable(tmpfile.fileno(), lines=True) # enable lines profiling function_foo() vmprof.disable() tmpfile.close() def walk(tree): assert len(tree.lines) >= len(tree.children) for v in six.itervalues(tree.children): walk(v) with open(tmpfile.name, "rb") as f: period, profiles, virtual_symbols, interp_name = read_prof_bit_by_bit(f) stats = Stats(profiles, virtual_symbols, interp_name) walk(stats.get_tree())
def main(): with tempfile.NamedTemporaryFile() as prof_file: vmprof.enable(prof_file.fileno(), 0.001) try: program = sys.argv[1] del sys.argv[1] # sys.argv = [args.program] + args.args runpy.run_path(program, run_name="__main__") except (KeyboardInterrupt, SystemExit): pass vmprof.disable() stats = vmprof.read_profile(prof_file.name, virtual_only=True) show(stats)
def main(): args = vmprof.cli.parse_args(sys.argv[1:]) # None means default on this platform native = None if args.no_native: native = False if args.web: output_mode = OUTPUT_WEB elif args.output: output_mode = OUTPUT_FILE else: output_mode = OUTPUT_CLI if output_mode == OUTPUT_FILE: prof_file = args.output prof_name = prof_file.name else: prof_file = tempfile.NamedTemporaryFile(delete=False) prof_name = prof_file.name vmprof.enable(prof_file.fileno(), args.period, args.mem, args.lines, native=native) if args.jitlog and _jitlog: fd = os.open(prof_name + '.jit', os.O_WRONLY | os.O_TRUNC | os.O_CREAT) _jitlog.enable(fd) # invoke the user program: try: sys.argv = [args.program] + args.args sys.path.insert(0, os.path.dirname(args.program)) runpy.run_path(args.program, run_name='__main__') except BaseException as e: if not isinstance(e, (KeyboardInterrupt, SystemExit)): raise # vmprof.disable() if args.jitlog and _jitlog: _jitlog.disable() prof_file.close() show_stats(prof_name, output_mode, args) if output_mode != OUTPUT_FILE: os.unlink(prof_name)
def profile_vmprof(name, env): if vmprof is None: print('vmprof not found. Please install vmprof and try again.') return func = create_bench(name, env) gc.collect() # # Based on: https://github.com/vmprof/vmprof-python/blob/master/vmprof/__main__.py # prof_file = tempfile.NamedTemporaryFile(delete=False) filename = prof_file.name vmprof.enable(prof_file.fileno()) try: for __ in range(1000000): func() except BaseException as e: if not isinstance(e, (KeyboardInterrupt, SystemExit)): raise vmprof.disable() service = Service('vmprof.com') service.post({ Service.FILE_CPU_PROFILE: filename, Service.FILE_JIT_PROFILE: filename + '.jit', 'argv': ' '.join(sys.argv[:]), 'VM': platform.python_implementation(), }) prof_file.close()
def __exit__(self, type, value, traceback): vmprof.disable() self.done = True
def run(): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # create the top-level parser # import argparse parser = argparse.ArgumentParser() parser.add_argument('--reactor', default=None, choices=['select', 'poll', 'epoll', 'kqueue', 'iocp'], help='Explicit Twisted reactor selection (optional).') parser.add_argument('--loglevel', default="info", choices=['none', 'error', 'warn', 'info', 'debug', 'trace'], help='Initial log level.') parser.add_argument('-c', '--cbdir', type=str, help="Crossbar.io node directory (required).") parser.add_argument('-n', '--node', type=str, help='Crossbar.io node ID (required).') parser.add_argument('-w', '--worker', type=str, help='Crossbar.io worker ID (required).') parser.add_argument('-r', '--realm', type=str, help='Crossbar.io node (management) realm (required).') parser.add_argument('-t', '--type', choices=['router', 'container'], help='Worker type (required).') parser.add_argument('--title', type=str, default=None, help='Worker process title to set (optional).') options = parser.parse_args() # make sure logging to something else than stdio is setup _first_ # from crossbar._logging import make_JSON_observer, cb_logging_aware, _stderr from crossbar._logging import make_logger, log_publisher, start_logging from crossbar._logging import set_global_log_level # Set the global log level set_global_log_level(options.loglevel) log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=_stderr) _stderr.flush() flo = make_JSON_observer(_stderr) log_publisher.addObserver(flo) start_logging() try: import setproctitle except ImportError: log.debug("Could not set worker process title (setproctitle not installed)") else: # set process title if requested to # if options.title: setproctitle.setproctitle(options.title) else: WORKER_TYPE_TO_TITLE = { 'router': 'crossbar-worker [router]', 'container': 'crossbar-worker [container]' } setproctitle.setproctitle(WORKER_TYPE_TO_TITLE[options.type].strip()) # we use an Autobahn utility to import the "best" available Twisted reactor # from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) from twisted.python.reflect import qual log.info("Worker running under {python}-{reactor}", python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1]) options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) from crossbar.worker.router import RouterWorkerSession from crossbar.worker.container import ContainerWorkerSession WORKER_TYPE_TO_CLASS = { 'router': RouterWorkerSession, 'container': ContainerWorkerSession } from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) log.warn("Connection to node controller lost.") WampWebSocketServerProtocol.connectionLost(self, reason) except: pass finally: # losing the connection to the node controller is fatal: # stop the reactor and exit with error log.info("No more controller connection; shutting down.") reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) try: reactor.stop() except ReactorNotRunning: pass try: # create a WAMP application session factory # from autobahn.twisted.wamp import ApplicationSessionFactory from autobahn.wamp.types import ComponentConfig session_config = ComponentConfig(realm=options.realm, extra=options) session_factory = ApplicationSessionFactory(session_config) session_factory.session = WORKER_TYPE_TO_CLASS[options.type] # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(session_factory, "ws://localhost", debug=False, debug_wamp=False) transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # if False: log.info("vmprof enabled.") import os import vmprof PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid()) outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(outfd, period=0.01) log.info("Entering event loop...") reactor.run() vmprof.disable() else: log.debug("Entering event loop...") reactor.run() except Exception as e: log.info("Unhandled exception: {}".format(e)) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)
def __exit__(self, type, value, traceback): vmprof.disable() self.tmpfile.close() # flushes the stream self.done = True
def _run_command_exec_worker(options, reactor=None, personality=None): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html#measuring-sub-processes MEASURING_COVERAGE = False if 'COVERAGE_PROCESS_START' in os.environ: try: import coverage except ImportError: pass else: # The following will read the environment variable COVERAGE_PROCESS_START, # and that should be set to the .coveragerc file: # # export COVERAGE_PROCESS_START=${PWD}/.coveragerc # coverage.process_startup() MEASURING_COVERAGE = True # we use an Autobahn utility to import the "best" available Twisted reactor from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) # make sure logging to something else than stdio is setup _first_ from crossbar._logging import make_JSON_observer, cb_logging_aware from txaio import make_logger, start_logging from twisted.logger import globalLogPublisher from twisted.python.reflect import qual log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=sys.__stderr__) sys.__stderr__.flush() flo = make_JSON_observer(sys.__stderr__) globalLogPublisher.addObserver(flo) # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # actually begin logging start_logging(None, options.loglevel) # get personality klass, eg "crossbar.personality.Personality" l = options.personality.split('.') personality_module, personality_klass = '.'.join(l[:-1]), l[-1] # now load the personality module and class _mod = importlib.import_module(personality_module) Personality = getattr(_mod, personality_klass) # get worker klass, eg "crossbar.worker.container.ContainerController" l = options.klass.split('.') worker_module, worker_klass = '.'.join(l[:-1]), l[-1] # now load the worker module and class _mod = importlib.import_module(worker_module) klass = getattr(_mod, worker_klass) log.info( 'Starting worker "{worker_id}" for node "{node_id}" with personality "{personality}" {worker_class}', worker_id=options.worker, node_id=options.node, personality=Personality.NAME, worker_class=hltype(klass), ) log.info( 'Running as PID {pid} on {python}-{reactor}', pid=os.getpid(), python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1], ) if MEASURING_COVERAGE: log.info(hl( 'Code coverage measurements enabled (coverage={coverage_version}).', color='green', bold=True), coverage_version=coverage.__version__) # set process title if requested to # try: import setproctitle except ImportError: log.debug( "Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle('crossbar-worker [{}]'.format( options.klass)) # node directory # options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) # set process title if requested to # try: import setproctitle except ImportError: log.debug( "Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle('crossbar-worker [{}]'.format( options.klass)) from twisted.internet.error import ConnectionDone from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): # the behavior here differs slightly whether we're shutting down orderly # or shutting down because of "issues" if isinstance(reason.value, ConnectionDone): was_clean = True else: was_clean = False try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) if was_clean: log.info("Connection to node controller closed cleanly") else: log.warn("Connection to node controller lost: {reason}", reason=reason) # give the WAMP transport a change to do it's thing WampWebSocketServerProtocol.connectionLost(self, reason) except: # we're in the process of shutting down .. so ignore .. pass finally: # after the connection to the node controller is gone, # the worker is "orphane", and should exit # determine process exit code if was_clean: exit_code = 0 else: exit_code = 1 # exit the whole worker process when the reactor has stopped reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code) # stop the reactor try: reactor.stop() except ReactorNotRunning: pass try: # define a WAMP application session factory # from autobahn.wamp.types import ComponentConfig def make_session(): session_config = ComponentConfig(realm=options.realm, extra=options) session = klass(config=session_config, reactor=reactor, personality=Personality) return session # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(make_session, u'ws://localhost') transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # if False: log.info("vmprof enabled.") import os import vmprof PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid()) outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(outfd, period=0.01) log.info(hl('Entering event reactor ...', color='cyan', bold=True)) reactor.run() vmprof.disable() else: log.info(hl('Entering event reactor ...', color='cyan', bold=True)) reactor.run() except Exception as e: log.info("Unhandled exception: {e}", e=e) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)
def before_reactor_stopped(): term_print('CROSSBAR[{}]:REACTOR_STOPPING'.format(options.worker)) if _HAS_VMPROF and options.vmprof and _vm_prof['outfd']: vmprof.disable() term_print('CROSSBAR[{}]:VMPROF_DISABLED'.format(options.worker))
def disable(self): if self.is_enabled: vmprof.disable() self.file.close() self.is_enabled = False
def test_basic(): tmpfile = tempfile.NamedTemporaryFile() vmprof.enable(tmpfile.fileno()) function_foo() vmprof.disable() assert "function_foo" in open(tmpfile.name).read()
def close_profile_file(): print("Closing VMProf!") vmprof.disable() print("VMProf closed!")
def before_reactor_stopped(): term_print('CROSSBAR:REACTOR_STOPPING') if _HAS_VMPROF and options.vmprof and _vm_prof['outfd']: vmprof.disable() term_print('CROSSBAR:VMPROF_DISABLED')
iters += 1 return iters * self.ITER_N, iters, done def test(): RUN_MS = 1000 RUNTIME = 30 print("Running for {} seconds ..\n".format(RUNTIME)) b = Burner() for i in range(RUNTIME): t, i, d = b.burn(RUN_MS) print("Actual run-time: {} / Requested run-time: {}, {} iterations. Total iterations: {}".format(d, RUN_MS, i, t)) if __name__ == '__main__': PROFILE_FILE = 'vmprof_cpuburn.dat' flags = os.O_RDWR | os.O_CREAT | os.O_TRUNC if sys.platform == 'win32': flags |= os.O_BINARY outfd = os.open(PROFILE_FILE, flags) vmprof.enable(outfd, period=0.01) test() vmprof.disable() print("\nProfile written to {}.".format(PROFILE_FILE)) print("To view the profile, run: vmprofshow {}".format(PROFILE_FILE))
def run(): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # create the top-level parser # import argparse parser = argparse.ArgumentParser() parser.add_argument('--reactor', default=None, choices=['select', 'poll', 'epoll', 'kqueue', 'iocp'], help='Explicit Twisted reactor selection (optional).') parser.add_argument('--loglevel', default="info", choices=['none', 'error', 'warn', 'info', 'debug', 'trace'], help='Initial log level.') parser.add_argument('-c', '--cbdir', type=str, help="Crossbar.io node directory (required).") parser.add_argument('-n', '--node', type=str, help='Crossbar.io node ID (required).') parser.add_argument('-w', '--worker', type=str, help='Crossbar.io worker ID (required).') parser.add_argument('-r', '--realm', type=str, help='Crossbar.io node (management) realm (required).') parser.add_argument('-t', '--type', choices=['router', 'container', 'websocket-testee'], help='Worker type (required).') parser.add_argument('--title', type=str, default=None, help='Worker process title to set (optional).') options = parser.parse_args() # make sure logging to something else than stdio is setup _first_ # from crossbar._logging import make_JSON_observer, cb_logging_aware, _stderr from crossbar._logging import make_logger, start_logging, set_global_log_level from twisted.logger import globalLogPublisher # Set the global log level set_global_log_level(options.loglevel) log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=_stderr) _stderr.flush() flo = make_JSON_observer(_stderr) globalLogPublisher.addObserver(flo) start_logging() try: import setproctitle except ImportError: log.debug("Could not set worker process title (setproctitle not installed)") else: # set process title if requested to # if options.title: setproctitle.setproctitle(options.title) else: WORKER_TYPE_TO_TITLE = { 'router': 'crossbar-worker [router]', 'container': 'crossbar-worker [container]', 'websocket-testee': 'crossbar-worker [websocket-testee]' } setproctitle.setproctitle(WORKER_TYPE_TO_TITLE[options.type].strip()) # we use an Autobahn utility to import the "best" available Twisted reactor # from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) from twisted.python.reflect import qual log.info("Worker running under {python}-{reactor}", python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1]) options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) from crossbar.worker.router import RouterWorkerSession from crossbar.worker.container import ContainerWorkerSession from crossbar.worker.testee import WebSocketTesteeWorkerSession WORKER_TYPE_TO_CLASS = { 'router': RouterWorkerSession, 'container': ContainerWorkerSession, 'websocket-testee': WebSocketTesteeWorkerSession } from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) log.warn("Connection to node controller lost.") WampWebSocketServerProtocol.connectionLost(self, reason) except: pass finally: # losing the connection to the node controller is fatal: # stop the reactor and exit with error log.info("No more controller connection; shutting down.") reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) try: reactor.stop() except ReactorNotRunning: pass try: # create a WAMP application session factory # from autobahn.twisted.wamp import ApplicationSessionFactory from autobahn.wamp.types import ComponentConfig session_config = ComponentConfig(realm=options.realm, extra=options) session_factory = ApplicationSessionFactory(session_config) session_factory.session = WORKER_TYPE_TO_CLASS[options.type] # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(session_factory, "ws://localhost", debug=False, debug_wamp=False) transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # if False: log.info("vmprof enabled.") import os import vmprof PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid()) outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(outfd, period=0.01) log.info("Entering event loop...") reactor.run() vmprof.disable() else: log.debug("Entering event loop...") reactor.run() except Exception as e: log.info("Unhandled exception: {}".format(e)) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)
# JSON serializer is always available serializers.append(serializer.JsonSerializer()) #serializers.append(serializer.JsonSerializer(batched=True)) # MsgPack serializers are optional if hasattr(serializer, 'MsgPackSerializer'): serializers.append(serializer.MsgPackSerializer()) #serializers.append(serializer.MsgPackSerializer(batched=True)) else: print("MsgPack not installed (pip install msgpack-python)") for test in tests: for ser in serializers: print("Running {} on serializer {} for {} seconds ..".format(test.__name__, ser.__class__, RUNTIME)) profile = PROFILE_FILE.format(test.__name__, ser.SERIALIZER_ID) outfd = os.open(profile, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(outfd, period=0.01) runtime, total_bytes, total_cnt = test(ser, RUNTIME) vmprof.disable() avg_msg_len = round(float(total_bytes) / float(total_cnt)) print("Processed {} messages in {} secs ({} total bytes serialized) at {} msgs/sec ({} bytes avg.)".format(total_cnt, round(runtime, 1), total_bytes, round(float(total_cnt) / runtime), avg_msg_len)) print("Profile written to {}.".format(profile)) print("To view the profile, run: vmprofshow {} --indent=2 --prune_percent=5".format(profile)) print
def run(): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # create the top-level parser # import argparse parser = argparse.ArgumentParser() parser.add_argument('--reactor', default=None, choices=['select', 'poll', 'epoll', 'kqueue', 'iocp'], help='Explicit Twisted reactor selection (optional).') parser.add_argument( '--loglevel', default="info", choices=['none', 'error', 'warn', 'info', 'debug', 'trace'], help='Initial log level.') parser.add_argument('-c', '--cbdir', type=six.text_type, help="Crossbar.io node directory (required).") parser.add_argument('-r', '--realm', type=six.text_type, help='Crossbar.io node (management) realm (required).') parser.add_argument('-t', '--type', choices=['router', 'container', 'websocket-testee'], help='Worker type (required).') parser.add_argument('-w', '--worker', type=six.text_type, help='Crossbar.io worker ID (required).') parser.add_argument('--title', type=six.text_type, default=None, help='Worker process title to set (optional).') parser.add_argument( '--expose_controller', type=bool, default=False, help= 'Expose node controller session to all components (this feature requires Crossbar.io Fabric extension).' ) parser.add_argument( '--expose_shared', type=bool, default=False, help= 'Expose a shared object to all components (this feature requires Crossbar.io Fabric extension).' ) options = parser.parse_args() # make sure logging to something else than stdio is setup _first_ # from crossbar._logging import make_JSON_observer, cb_logging_aware from txaio import make_logger, start_logging from twisted.logger import globalLogPublisher log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=sys.__stderr__) sys.__stderr__.flush() flo = make_JSON_observer(sys.__stderr__) globalLogPublisher.addObserver(flo) start_logging(None, options.loglevel) # we use an Autobahn utility to import the "best" available Twisted reactor # from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) WORKER_TYPE_TO_TITLE = { 'router': 'Router', 'container': 'Container', 'websocket-testee': 'WebSocket Testee' } from twisted.python.reflect import qual log.info( '{worker_title} worker "{worker_id}" process {pid} starting on {python}-{reactor} ..', worker_title=WORKER_TYPE_TO_TITLE[options.type], worker_id=options.worker, pid=os.getpid(), python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1]) # set process title if requested to # try: import setproctitle except ImportError: log.debug( "Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: WORKER_TYPE_TO_PROCESS_TITLE = { 'router': 'crossbar-worker [router]', 'container': 'crossbar-worker [container]', 'websocket-testee': 'crossbar-worker [websocket-testee]' } setproctitle.setproctitle( WORKER_TYPE_TO_PROCESS_TITLE[options.type].strip()) # node directory # options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) from crossbar.worker.router import RouterWorkerSession from crossbar.worker.container import ContainerWorkerSession from crossbar.worker.testee import WebSocketTesteeWorkerSession WORKER_TYPE_TO_CLASS = { 'router': RouterWorkerSession, 'container': ContainerWorkerSession, 'websocket-testee': WebSocketTesteeWorkerSession } from twisted.internet.error import ConnectionDone from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): # the behavior here differs slightly whether we're shutting down orderly # or shutting down because of "issues" if isinstance(reason.value, ConnectionDone): was_clean = True else: was_clean = False try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) if was_clean: log.info("Connection to node controller closed cleanly") else: log.warn("Connection to node controller lost: {reason}", reason=reason) # give the WAMP transport a change to do it's thing WampWebSocketServerProtocol.connectionLost(self, reason) except: # we're in the process of shutting down .. so ignore .. pass finally: # after the connection to the node controller is gone, # the worker is "orphane", and should exit # determine process exit code if was_clean: exit_code = 0 else: exit_code = 1 # exit the whole worker process when the reactor has stopped reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code) # stop the reactor try: reactor.stop() except ReactorNotRunning: pass try: # create a WAMP application session factory # from autobahn.twisted.wamp import ApplicationSessionFactory from autobahn.wamp.types import ComponentConfig session_config = ComponentConfig(realm=options.realm, extra=options) session_factory = ApplicationSessionFactory(session_config) session_factory.session = WORKER_TYPE_TO_CLASS[options.type] # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(session_factory, u'ws://localhost') transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # if False: log.info("vmprof enabled.") import os import vmprof PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid()) outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(outfd, period=0.01) log.info("Entering event loop...") reactor.run() vmprof.disable() else: log.debug("Entering event loop...") reactor.run() except Exception as e: log.info("Unhandled exception: {e}", e=e) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)
def _disable_profiler(self): log.info("Disable Profiler - wrote to {}".format(PROFILER_FILE_NAME)) vmprof.disable() os.close(self._profiler_fd) self._profiler_running = False
def test_basic(): tmpfile = tempfile.NamedTemporaryFile() vmprof.enable(tmpfile.fileno()) function_foo() vmprof.disable() assert b"function_foo" in open(tmpfile.name, 'rb').read()
def _run_command_exec_worker(options, reactor=None, personality=None): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html#measuring-sub-processes MEASURING_COVERAGE = False if 'COVERAGE_PROCESS_START' in os.environ: try: import coverage except ImportError: pass else: # The following will read the environment variable COVERAGE_PROCESS_START, # and that should be set to the .coveragerc file: # # export COVERAGE_PROCESS_START=${PWD}/.coveragerc # coverage.process_startup() MEASURING_COVERAGE = True # we use an Autobahn utility to import the "best" available Twisted reactor from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) # make sure logging to something else than stdio is setup _first_ from crossbar._logging import make_JSON_observer, cb_logging_aware from txaio import make_logger, start_logging from twisted.logger import globalLogPublisher from twisted.python.reflect import qual log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=sys.__stderr__) sys.__stderr__.flush() flo = make_JSON_observer(sys.__stderr__) globalLogPublisher.addObserver(flo) # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # actually begin logging start_logging(None, options.loglevel) # get personality klass, eg "crossbar.personality.Personality" l = options.personality.split('.') personality_module, personality_klass = '.'.join(l[:-1]), l[-1] # now load the personality module and class _mod = importlib.import_module(personality_module) Personality = getattr(_mod, personality_klass) # get worker klass, eg "crossbar.worker.container.ContainerController" l = options.klass.split('.') worker_module, worker_klass = '.'.join(l[:-1]), l[-1] # now load the worker module and class _mod = importlib.import_module(worker_module) klass = getattr(_mod, worker_klass) log.info( 'Starting worker "{worker_id}" for node "{node_id}" with personality "{personality}" {worker_class}', worker_id=options.worker, node_id=options.node, personality=Personality.NAME, worker_class=hltype(klass), ) log.info( 'Running as PID {pid} on {python}-{reactor}', pid=os.getpid(), python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1], ) if MEASURING_COVERAGE: log.info(hl('Code coverage measurements enabled (coverage={coverage_version}).', color='green', bold=True), coverage_version=coverage.__version__) # set process title if requested to # try: import setproctitle except ImportError: log.debug("Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle('crossbar-worker [{}]'.format(options.klass)) # node directory # options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) # set process title if requested to # try: import setproctitle except ImportError: log.debug("Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle( 'crossbar-worker [{}]'.format(options.klass) ) from twisted.internet.error import ConnectionDone from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): # the behavior here differs slightly whether we're shutting down orderly # or shutting down because of "issues" if isinstance(reason.value, ConnectionDone): was_clean = True else: was_clean = False try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) if was_clean: log.info("Connection to node controller closed cleanly") else: log.warn("Connection to node controller lost: {reason}", reason=reason) # give the WAMP transport a change to do it's thing WampWebSocketServerProtocol.connectionLost(self, reason) except: # we're in the process of shutting down .. so ignore .. pass finally: # after the connection to the node controller is gone, # the worker is "orphane", and should exit # determine process exit code if was_clean: exit_code = 0 else: exit_code = 1 # exit the whole worker process when the reactor has stopped reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code) # stop the reactor try: reactor.stop() except ReactorNotRunning: pass try: # define a WAMP application session factory # from autobahn.wamp.types import ComponentConfig def make_session(): session_config = ComponentConfig(realm=options.realm, extra=options) session = klass(config=session_config, reactor=reactor, personality=Personality) return session # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(make_session, u'ws://localhost') transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # if False: log.info("vmprof enabled.") import os import vmprof PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid()) outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(outfd, period=0.01) log.info(hl('Entering event reactor ...', color='cyan', bold=True)) reactor.run() vmprof.disable() else: log.info(hl('Entering event reactor ...', color='cyan', bold=True)) reactor.run() except Exception as e: log.info("Unhandled exception: {e}", e=e) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)