def command_start(args): procfile_path = _procfile_path(args.app_root, args.procfile) procfile = _procfile(procfile_path) port = int(os.environ.get('PORT', args.port)) concurrency = _parse_concurrency(args.concurrency) env = _read_env(procfile_path, args.env) quiet = _parse_quiet(args.quiet) if args.processes: processes = compat.OrderedDict() for name in args.processes: try: processes[name] = procfile.processes[name] except KeyError: raise CommandError("Process type '{0}' does not exist in Procfile".format(name)) else: processes = procfile.processes manager = Manager() for p in environ.expand_processes(processes, concurrency=concurrency, env=env, quiet=quiet, port=port): e = os.environ.copy() e.update(p.env) manager.add_process(p.name, p.cmd, quiet=p.quiet, env=e) manager.loop() sys.exit(manager.returncode)
def devserver(environment, workers, port): os.environ.setdefault('FLASK_DEBUG', '1') os.environ['NODE_ENV'] = environment click.echo('Launching Zeus on http://localhost:{}'.format(port)) # TODO(dcramer): pass required attributes to 'run' directly instead # of relying on FLASK_DEBUG daemons = [ ('web', ['zeus', 'run', '--port={}'.format(port)]), ('webpack', ['node_modules/.bin/webpack', '--watch', '--config=config/webpack.config.js']), ] if workers: daemons.append( ('worker', ['zeus', 'worker', '--cron', '--log-level=INFO'])) cwd = os.path.realpath(os.path.join( os.path.dirname(__file__), os.pardir, os.pardir)) manager = Manager() for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, cwd=cwd, ) manager.loop() sys.exit(manager.returncode)
def devserver(web, worker, sched, webpack): """ Run the flask development server, workers, and scheduler """ try: from honcho.manager import Manager except ImportError: raise click.ClickException( 'cannot import honcho: did you run `pip install -e .` ?' ) os.environ['PYTHONUNBUFFERED'] = 'true' m = Manager() if web: m.add_process('web', 'python wsgi.py') if worker: m.add_process('worker', 'python worker.py') if sched: m.add_process('sched', 'python scheduler.py') if webpack: m.add_process('webpack', 'npm run start') m.loop() sys.exit(m.returncode)
def handle(self, *args, **options): m = Manager() m.add_process('web', './manage.py runserver') m.add_process('celery', 'celery -A config worker --beat -l info') m.loop() sys.exit(m.returncode)
def command_start(args): procfile_path = _procfile_path(args.app_root, _choose_procfile(args)) procfile = _procfile(procfile_path) concurrency = _parse_concurrency(args.concurrency) env = _read_env(args.app_root, args.env) quiet = _parse_quiet(args.quiet) port = _choose_port(args, env) if args.processes: processes = OrderedDict() for name in args.processes: try: processes[name] = procfile.processes[name] except KeyError: raise CommandError("Process type '{0}' does not exist in Procfile".format(name)) else: processes = procfile.processes manager = Manager(Printer(sys.stdout, colour=(not args.no_colour), prefix=(not args.no_prefix))) for p in environ.expand_processes(processes, concurrency=concurrency, env=env, quiet=quiet, port=port): e = os.environ.copy() e.update(p.env) manager.add_process(p.name, p.cmd, quiet=p.quiet, env=e) manager.loop() sys.exit(manager.returncode)
def main(): src_path = os.path.dirname(os.path.realpath(__file__)) base_path = os.path.normpath( os.path.join(os.path.dirname(__file__), os.pardir, os.pardir) ) start = [ ('api', ['python', 'runner.py', 'api'], src_path), ('web', ['python', 'runner.py', 'web'], src_path), ('play_server', ['python', 'runner.py', 'play_server'], src_path), ('worker', ['python', 'runner.py', 'worker'], src_path), ('npm', ['npm', 'run', 'start'], base_path), ] if watchdog_installed: start.append( ('play_scan_watch', ['python', 'runner.py', 'play_scan_watch'], src_path), ) manager = Manager() for name, cmd, cwd in start: manager.add_process( name, list2cmdline(cmd), quiet=False, cwd=cwd, ) manager.loop() sys.exit(manager.returncode)
def command_start(args): procfile_path = _procfile_path(args.app_root, args.procfile) procfile = _procfile(procfile_path) port = int(os.environ.get('PORT', args.port)) concurrency = _parse_concurrency(args.concurrency) env = _read_env(procfile_path, args.env) quiet = _parse_quiet(args.quiet) if args.processes: processes = compat.OrderedDict() for name in args.processes: try: processes[name] = procfile.processes[name] except KeyError: raise CommandError( "Process type '{0}' does not exist in Procfile".format( name)) else: processes = procfile.processes manager = Manager() for p in environ.expand_processes(processes, concurrency=concurrency, env=env, quiet=quiet, port=port): e = os.environ.copy() e.update(p.env) manager.add_process(p.name, p.cmd, quiet=p.quiet, env=e) manager.loop() sys.exit(manager.returncode)
def devserver(https, web, ws, worker, assets, beat): """ Run a development server. This command will start a development instance of h, consisting of a web application, Celery worker, and websocket server. It will also start a process which will watch and build the frontend assets. By default, the webserver will be accessible at: http://localhost:5000 You can also pass the `--https` flag, which will look for a TLS certificate and key in PEM format in the current directory, in files called: .tlscert.pem .tlskey.pem If you use this flag, the webserver will be accessible at: https://localhost:5000 If you wish this to be the default behaviour, you can set the USE_HTTPS environment variable. """ try: from honcho.manager import Manager except ImportError: raise click.ClickException('cannot import honcho: did you run `pip install -r requirements-dev.in` yet?') os.environ['PYTHONUNBUFFERED'] = 'true' if https: gunicorn_args = '--certfile=.tlscert.pem --keyfile=.tlskey.pem' os.environ['APP_URL'] = 'https://localhost:5000' os.environ['WEBSOCKET_URL'] = 'wss://localhost:5001/ws' else: gunicorn_args = '' os.environ['APP_URL'] = 'http://localhost:5000' os.environ['WEBSOCKET_URL'] = 'ws://localhost:5001/ws' m = Manager() if web: m.add_process('web', 'gunicorn --name web --reload --paste conf/development-app.ini %s' % gunicorn_args) if ws: m.add_process('ws', 'gunicorn --name websocket --reload --paste conf/development-websocket.ini %s' % gunicorn_args) if worker: m.add_process('worker', 'hypothesis --dev celery worker --autoreload') if beat: m.add_process('beat', 'hypothesis --dev celery beat') if assets: m.add_process('assets', 'gulp watch') m.loop() sys.exit(m.returncode)
def inner_run(self, *args, **options): # Check a handler is registered for http reqs; if not, add default one self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] self.channel_layer.router.check_default( http_consumer=ViewConsumer(), ) self.addr = self.get_addr(options['addr']) self.port = self.get_port(options['port']) # Run checks self.stdout.write("Performing system checks...\n\n") self.check(display_num_errors=True) self.check_migrations() # Print helpful text quit_command = 'CTRL-BREAK' if sys.platform == 'win32' else 'CONTROL-C' self.stdout.write(( "\n" "otree-core version %(otree_version)s, " "Django version %(django_version)s, " "using settings %(settings)r\n" "Starting web server at http://%(addr)s:%(port)s/\n" # "Channel layer %(layer)s\n" "Quit the server with %(quit_command)s.\n" ) % { "django_version": self.get_version(), "otree_version": otree.__version__, "settings": settings.SETTINGS_MODULE, "addr": '[%s]' % self.addr if self._raw_ipv6 else self.addr, "port": self.port, "quit_command": quit_command, "layer": self.channel_layer, }) self.stdout.flush() # Launch server in 'main' thread. Signals are disabled as it's still # actually a subthread under the autoreloader. self.logger.debug("Daphne running, listening on %s:%s", self.addr, self.port) manager = Manager() daphne_cmd = 'daphne otree.asgi:channel_layer -b {} -p {}'.format( self.addr, self.port ) print(daphne_cmd) manager.add_process('daphne', daphne_cmd, env=self.get_env(options)) for i in range(3): manager.add_process( 'worker{}'.format(i), 'otree runworker', env=self.get_env(options)) manager.loop() sys.exit(manager.returncode)
def inner_run(self, *args, **options): # Check a handler is registered for http reqs; if not, add default one self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] self.channel_layer.router.check_default(http_consumer=ViewConsumer(), ) self.addr = self.get_addr(options['addr']) self.port = self.get_port(options['port']) # Run checks self.stdout.write("Performing system checks...\n\n") self.check(display_num_errors=True) self.check_migrations() # Print helpful text quit_command = 'CTRL-BREAK' if sys.platform == 'win32' else 'CONTROL-C' self.stdout.write( ( "\n" "otree-core version %(otree_version)s, " "Django version %(django_version)s, " "using settings %(settings)r\n" "Starting web server at http://%(addr)s:%(port)s/\n" # "Channel layer %(layer)s\n" "Quit the server with %(quit_command)s.\n") % { "django_version": self.get_version(), "otree_version": otree.__version__, "settings": settings.SETTINGS_MODULE, "addr": '[%s]' % self.addr if self._raw_ipv6 else self.addr, "port": self.port, "quit_command": quit_command, "layer": self.channel_layer, }) self.stdout.flush() # Launch server in 'main' thread. Signals are disabled as it's still # actually a subthread under the autoreloader. self.logger.debug("Daphne running, listening on %s:%s", self.addr, self.port) manager = Manager() daphne_cmd = 'daphne otree.asgi:channel_layer -b {} -p {}'.format( self.addr, self.port) print(daphne_cmd) manager.add_process('daphne', daphne_cmd, env=self.get_env(options)) for i in range(3): manager.add_process('worker{}'.format(i), 'otree runworker', env=self.get_env(options)) manager.loop() sys.exit(manager.returncode)
def handle(self, *args, **options): manager = Manager() for i in range(options['num_workers']): manager.add_process('worker{}'.format(i), 'otree runworker', quiet=False, env=self.get_env(options)) manager.loop() sys.exit(manager.returncode)
def tests(self): """ $ python -m saai.runner tests :return: """ m = Manager() m.add_process('nlu_en', './startup.run nlu en 2001') m.add_process('nlu_zh', './startup.run nlu zh 2002') m.loop() sys.exit(m.returncode)
def devserver(environment, workers, port, ngrok, ngrok_domain, pubsub, pubsub_port): os.environ.setdefault('FLASK_DEBUG', '1') os.environ['NODE_ENV'] = environment if pubsub: os.environ['PUBSUB_ENDPOINT'] = 'http://localhost:{}'.format( pubsub_port) if ngrok: root_url = 'https://{}.ngrok.io'.format(ngrok_domain) os.environ['SSL'] = '1' os.environ['SERVER_NAME'] = '{}.ngrok.io'.format(ngrok_domain) else: root_url = 'http://localhost:{}'.format(port) click.echo('Launching Zeus on {}'.format(root_url)) # TODO(dcramer): pass required attributes to 'run' directly instead # of relying on FLASK_DEBUG daemons = [ ('web', ['zeus', 'run', '--port={}'.format(port)]), ('webpack', [ 'node_modules/.bin/webpack', '--watch', '--config=config/webpack.config.js' ]), ] if pubsub: daemons.append( ('pubsub', ['zeus', 'pubsub', '--port={}'.format(pubsub_port)])) if workers: daemons.append( ('worker', ['zeus', 'worker', '--cron', '--log-level=INFO'])) if ngrok: daemons.append(('ngrok', [ 'ngrok', 'http', '-subdomain={}'.format(ngrok_domain), str(port) ])) cwd = os.path.realpath( os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) manager = Manager() for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, cwd=cwd, ) manager.loop() sys.exit(manager.returncode)
def handle(self, *args, **options): manager = Manager() for i in range(options['num_workers']): manager.add_process( 'worker{}'.format(i), 'otree runworker', quiet=False, env=self.get_env(options)) manager.loop() sys.exit(manager.returncode)
def devserver(environment, workers, port, ngrok, ngrok_domain, pubsub, pubsub_port): os.environ.setdefault("FLASK_DEBUG", "1") os.environ["NODE_ENV"] = environment if pubsub: os.environ["PUBSUB_ENDPOINT"] = "http://localhost:{}".format( pubsub_port) if ngrok: root_url = "https://{}.ngrok.io".format(ngrok_domain) os.environ["SSL"] = "1" os.environ["SERVER_NAME"] = "{}.ngrok.io".format(ngrok_domain) else: root_url = "http://localhost:{}".format(port) click.echo("Launching Zeus on {}".format(root_url)) # TODO(dcramer): pass required attributes to 'run' directly instead # of relying on FLASK_DEBUG daemons = [ ("web", ["zeus", "run", "--port={}".format(port)]), ( "webpack", [ "node_modules/.bin/webpack", "--watch", "--config=config/webpack.config.js", ], ), ] if pubsub: daemons.append( ("pubsub", ["zeus", "pubsub", "--port={}".format(pubsub_port)])) if workers: daemons.append( ("worker", ["zeus", "worker", "--cron", "--log-level=INFO"])) if ngrok: daemons.append(( "ngrok", ["ngrok", "http", "-subdomain={}".format(ngrok_domain), str(port)], )) cwd = os.path.realpath( os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) manager = Manager() for name, cmd in daemons: manager.add_process(name, list2cmdline(cmd), quiet=False, cwd=cwd) manager.loop() sys.exit(manager.returncode)
def web(host, port, processes, threads): os.environ["PYTHONUNBUFFERED"] = "true" daemons = [( "web", [ "uwsgi", "--master", "--vacuum", "--enable-threads", # "--lazy-apps", "--single-interpreter", "--http={}:{}".format(host, port), "--processes={}".format(processes), "--threads={}".format(threads), "--log-x-forwarded-for", "--buffer-size=32768", "--post-buffering=65536", "--need-app", "--harakiri=120", "--harakiri-verbose", "--thunder-lock", "--vacuum", "--home={}".format(sys.prefix), "--auto-procname", "--procname-prefix-spaced=[zeus]", "--module=zeus.app:app", "--die-on-term", "--ignore-sigpipe", "--ignore-write-errors", "--disable-write-exception", # XXX(dcramer): this is disabled as our version of uwsgi doesnt # know what it means # '--wsgi-manage-chunked-input', ], )] cwd = os.path.realpath( os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) manager = Manager() for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, cwd=cwd, # env=os.environ, ) manager.loop() sys.exit(manager.returncode)
def exec_spec(self, langs): """ $ python -m saai.runner exec_spec zh,en,ja $ python -m saai.runner exec_spec zh,en,ja,de :param langs: :return: """ m = Manager() for lang in langs: port = self.nlu_servants[lang] m.add_process(f'nlu_{lang}', f'./startup.run nlu {lang} {port}') m.loop() sys.exit(m.returncode)
def web(host, port, processes, threads): os.environ['PYTHONUNBUFFERED'] = 'true' daemons = [ ( 'web', [ 'uwsgi', '--master', '--enable-threads', '--lazy-apps', '--single-interpreter', '--http={}:{}'.format(host, port), '--processes={}'.format(processes), '--threads={}'.format(threads), '--log-x-forwarded-for', '--buffer-size=32768', '--post-buffering=65536', '--need-app', '--disable-logging', '--thunder-lock', '--vacuum', '--home={}'.format(sys.prefix), '--auto-procname', '--procname-prefix-spaced=[zeus]', '--module=zeus.app:app', '--die-on-term', # XXX(dcramer): this is disabled as our version of uwsgi doesnt # know what it means # '--wsgi-manage-chunked-input', ]), ] cwd = os.path.realpath( os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) manager = Manager() for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, cwd=cwd, # env=os.environ, ) manager.loop() sys.exit(manager.returncode)
def devserver(bootstrap, workers): "Starts all Snuba processes for local development." import os import sys from subprocess import list2cmdline, call from honcho.manager import Manager os.environ['PYTHONUNBUFFERED'] = '1' if bootstrap: cmd = ['snuba', 'bootstrap', '--force'] if not workers: cmd.append('--no-kafka') returncode = call(cmd) if returncode > 0: sys.exit(returncode) daemons = [ ('api', [ 'uwsgi', '--master', '--manage-script-name', '--wsgi-file', 'snuba/views.py', '--http', '0.0.0.0:1218', '--http-keepalive', '--need-app', '--die-on-term', ]), ] if not workers: os.execvp(daemons[0][1][0], daemons[0][1]) daemons += [ ('transaction-consumer', ['snuba', 'consumer', '--auto-offset-reset=latest', '--log-level=debug', '--dataset=transactions', '--consumer-group=transactions_group']), ('consumer', ['snuba', 'consumer', '--auto-offset-reset=latest', '--log-level=debug']), ('replacer', ['snuba', 'replacer', '--auto-offset-reset=latest', '--log-level=debug']), ] manager = Manager() for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, ) manager.loop() sys.exit(manager.returncode)
def main(): parser = argparse.ArgumentParser() parser.add_argument('specdir', help="directory containging resource type definitions") parser.add_argument('-c', '--appconfig', default='', help="app configuration file") parser.add_argument('-d', '--basedir', default=DEFAULT_BASE_DIR, help="path to config files and etc.") args = parser.parse_args() set_up_logging() set_up_configs(args.basedir) set_up_gluu_db(args.basedir) haproxy_cfg = os.path.abspath(os.path.join(args.basedir, 'etc', 'haproxy.cfg')) qvarn_conf = os.path.abspath(os.path.join(args.basedir, 'etc', 'qvarn.conf')) gluu_db_uri = 'sqlite:///' + os.path.abspath(os.path.join(args.basedir, 'var', 'gluu.db')) logger.info('gluu db: %s', gluu_db_uri) m = Manager() uwsgi = ' '.join([ 'uwsgi', '--http-socket 127.0.0.1:9000', '--wsgi-file', pres.resource_filename('qvarntesting', 'wsgi.py'), '--pyargv "%s --config %s"' % (args.specdir, qvarn_conf), '--master', # Default buffer-size is 4k, and I already hit that limit. '--buffer-size 10240', '--py-autoreload 1', ]) logger.info(uwsgi) m.add_process('uwsgi', uwsgi, env=env({ 'APP_CONFIG': args.appconfig, 'DATABASE_URL': gluu_db_uri, })) haproxy = 'haproxy -f %s -C %s -db' % ( haproxy_cfg, pres.resource_filename('qvarntesting', 'config/certs'), ) logger.info(haproxy) m.add_process('haproxy', haproxy) m.loop() return m.returncode
def main(): path = os.path.dirname(os.path.realpath(__file__)) start = [ ('twitch_bot', ['python', 'runner.py', 'twitch-bot'], path), ('discord', ['python', 'runner.py', 'discord'], path), ('web', ['python', 'runner.py', 'web'], path), ] manager = Manager() for name, cmd, cwd in start: manager.add_process( name, list2cmdline(cmd), quiet=False, cwd=cwd, ) manager.loop()
def devserver(port, workers, ngrok, ngrok_domain): os.environ.setdefault('FLASK_DEBUG', '1') if ngrok: os.environ['SERVER_NAME'] = '{}.ngrok.io'.format(ngrok_domain) root_url = 'https://{}'.format(os.environ['SERVER_NAME']) os.environ['SSL'] = '1' else: root_url = 'http://localhost:{}'.format(port) click.echo('Launching Cuckoo on {}'.format(root_url)) daemons = [ ('web', ['cuckoo', 'run', '--port={}'.format(port)]), ] if workers: daemons.append( ('worker', ['cuckoo', 'worker', '--cron', '--log-level=INFO']), ) if ngrok: daemons.append( ('ngrok', ['ngrok', 'http', '-subdomain={}'.format(ngrok_domain), str(port)]) ) cwd = os.path.realpath(os.path.join( os.path.dirname(__file__), os.pardir, os.pardir)) manager = Manager() for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, cwd=cwd, ) manager.loop() sys.exit(manager.returncode)
def handle(self, *args, **options): manager = Manager() manager.add_process( 'web', 'otree runweb', quiet=False, env=self.get_env(options)) manager.add_process( 'channels', 'otree runchannelsworker', quiet=False, env=self.get_env(options)) manager.add_process( 'celery', 'otree runceleryworker', quiet=False, env=self.get_env(options)) manager.loop() sys.exit(manager.returncode)
def run(self): from subprocess import list2cmdline from honcho.manager import Manager as HonchoManager os.environ['PYTHONUNBUFFERED'] = 'true' daemons = [ ('server', ['./manage.py', 'runsocketserver']), ('worker', ['./manage.py', 'celery']), ('cron', ['./manage.py', 'beat']), ('sass', ['./opinew.sh', 'sass_watcher']), ] cwd = os.path.abspath(os.path.dirname(__file__)) honcho_manager = HonchoManager() for name, cmd in daemons: honcho_manager.add_process( name, list2cmdline(cmd), quiet=False, cwd=cwd, ) honcho_manager.loop() sys.exit(honcho_manager.returncode)
def main(): src_path = os.path.dirname(os.path.realpath(__file__)) start = [ ('api', ['python', 'runner.py', 'api'], src_path), ('web', ['python', 'runner.py', 'web'], src_path), ('play-server', ['python', 'runner.py', 'play-server'], src_path), ('worker', ['python', 'runner.py', 'worker'], src_path), ] if watchdog_installed: start.append( ('play-scan-watch', ['python', 'runner.py', 'play-scan-watch' ], src_path), ) manager = Manager() for name, cmd, cwd in start: manager.add_process( name, list2cmdline(cmd), quiet=False, cwd=cwd, ) manager.loop()
def devserver(reload, watchers, workers, browser_reload, environment, bind): "Starts a lightweight web server for development." if ':' in bind: host, port = bind.split(':', 1) port = int(port) else: host = bind port = None import os os.environ['SENTRY_ENVIRONMENT'] = environment from django.conf import settings from sentry import options from sentry.services.http import SentryHTTPServer url_prefix = options.get('system.url-prefix', '') needs_https = url_prefix.startswith('https://') has_https = False if needs_https: from subprocess import check_output try: check_output(['which', 'https']) has_https = True except Exception: has_https = False from sentry.runner.initializer import show_big_error show_big_error([ 'missing `https` on your `$PATH`, but https is needed', '`$ brew install mattrobenolt/stuff/https`', ]) uwsgi_overrides = { # Make sure we don't try and use uwsgi protocol 'protocol': 'http', # Make sure we reload really quickly for local dev in case it # doesn't want to shut down nicely on it's own, NO MERCY 'worker-reload-mercy': 2, # We need stdin to support pdb in devserver 'honour-stdin': True, } if reload: uwsgi_overrides['py-autoreload'] = 1 daemons = [] if watchers: daemons += settings.SENTRY_WATCHERS if workers: if settings.CELERY_ALWAYS_EAGER: raise click.ClickException('Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.') daemons += [ ('worker', ['sentry', 'run', 'worker', '-c', '1', '--autoreload']), ('cron', ['sentry', 'run', 'cron', '--autoreload']), ] if needs_https and has_https: from six.moves.urllib.parse import urlparse parsed_url = urlparse(url_prefix) https_port = six.text_type(parsed_url.port or 443) https_host = parsed_url.hostname # Determine a random port for the backend http server import socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind((host, 0)) port = s.getsockname()[1] s.close() bind = '%s:%d' % (host, port) daemons += [ ('https', ['https', '-host', https_host, '-listen', host + ':' + https_port, bind]), ] # A better log-format for local dev when running through honcho, # but if there aren't any other daemons, we don't want to override. if daemons: uwsgi_overrides['log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)' else: uwsgi_overrides['log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)' server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides) # If we don't need any other daemons, just launch a normal uwsgi webserver # and avoid dealing with subprocesses if not daemons: return server.run() import sys from subprocess import list2cmdline from honcho.manager import Manager os.environ['PYTHONUNBUFFERED'] = 'true' if browser_reload: os.environ['WEBPACK_LIVERELOAD'] = '1' # Make sure that the environment is prepared before honcho takes over # This sets all the appropriate uwsgi env vars, etc server.prepare_environment() daemons += [ ('server', ['sentry', 'run', 'web']), ] cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir)) manager = Manager() for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, cwd=cwd, ) manager.loop() sys.exit(manager.returncode)
def devserver(reload, watchers, workers, experimental_spa, styleguide, prefix, environment, bind): "Starts a lightweight web server for development." if ":" in bind: host, port = bind.split(":", 1) port = int(port) else: host = bind port = None import os os.environ["SENTRY_ENVIRONMENT"] = environment from django.conf import settings from sentry import options from sentry.services.http import SentryHTTPServer url_prefix = options.get("system.url-prefix", "") parsed_url = urlparse(url_prefix) # Make sure we're trying to use a port that we can actually bind to needs_https = parsed_url.scheme == "https" and (parsed_url.port or 443) > 1024 has_https = False if needs_https: from subprocess import check_output try: check_output(["which", "https"]) has_https = True except Exception: has_https = False from sentry.runner.initializer import show_big_error show_big_error( [ "missing `https` on your `$PATH`, but https is needed", "`$ brew install mattrobenolt/stuff/https`", ] ) uwsgi_overrides = { # Make sure we don't try and use uwsgi protocol "protocol": "http", # Make sure we reload really quickly for local dev in case it # doesn't want to shut down nicely on it's own, NO MERCY "worker-reload-mercy": 2, # We need stdin to support pdb in devserver "honour-stdin": True, # accept ridiculously large files "limit-post": 1 << 30, # do something with chunked "http-chunked-input": True, "thunder-lock": False, "timeout": 600, "harakiri": 600, } if reload: uwsgi_overrides["py-autoreload"] = 1 daemons = [] if experimental_spa: os.environ["SENTRY_EXPERIMENTAL_SPA"] = "1" if not watchers: click.secho( "Using experimental SPA mode without watchers enabled has no effect", err=True, fg="yellow", ) # We proxy all requests through webpacks devserver on the configured port. # The backend is served on port+1 and is proxied via the webpack # configuration. if watchers: daemons += settings.SENTRY_WATCHERS proxy_port = port port = port + 1 os.environ["SENTRY_WEBPACK_PROXY_PORT"] = "%s" % proxy_port os.environ["SENTRY_BACKEND_PORT"] = "%s" % port # Replace the webpack watcher with the drop-in webpack-dev-server webpack_config = next(w for w in daemons if w[0] == "webpack")[1] webpack_config[0] = os.path.join( *os.path.split(webpack_config[0])[0:-1] + ("webpack-dev-server",) ) daemons = [w for w in daemons if w[0] != "webpack"] + [("webpack", webpack_config)] if workers: if settings.CELERY_ALWAYS_EAGER: raise click.ClickException( "Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers." ) daemons += [ ("worker", ["sentry", "run", "worker", "-c", "1", "--autoreload"]), ("cron", ["sentry", "run", "cron", "--autoreload"]), ] from sentry import eventstream if eventstream.requires_post_process_forwarder(): daemons += [ ( "relay", [ "sentry", "run", "post-process-forwarder", "--loglevel=debug", "--commit-batch-size=1", ], ) ] if needs_https and has_https: https_port = six.text_type(parsed_url.port) https_host = parsed_url.hostname # Determine a random port for the backend http server import socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind((host, 0)) port = s.getsockname()[1] s.close() bind = "%s:%d" % (host, port) daemons += [ ("https", ["https", "-host", https_host, "-listen", host + ":" + https_port, bind]) ] # A better log-format for local dev when running through honcho, # but if there aren't any other daemons, we don't want to override. if daemons: uwsgi_overrides["log-format"] = '"%(method) %(uri) %(proto)" %(status) %(size)' else: uwsgi_overrides["log-format"] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)' server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides) # If we don't need any other daemons, just launch a normal uwsgi webserver # and avoid dealing with subprocesses if not daemons: return server.run() import sys from subprocess import list2cmdline from honcho.manager import Manager from honcho.printer import Printer os.environ["PYTHONUNBUFFERED"] = "true" # Make sure that the environment is prepared before honcho takes over # This sets all the appropriate uwsgi env vars, etc server.prepare_environment() daemons += [("server", ["sentry", "run", "web"])] if styleguide: daemons += [("storybook", ["./bin/yarn", "storybook"])] cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir)) manager = Manager(Printer(prefix=prefix)) for name, cmd in daemons: manager.add_process(name, list2cmdline(cmd), quiet=False, cwd=cwd) manager.loop() sys.exit(manager.returncode)
class TestManager(object): @pytest.fixture(autouse=True) def printer(self): # noqa self.p = FakePrinter() self.m = Manager(printer=self.p) self.m._env = FakeEnv() def run_history(self, name, wait=True): self.h = Harness(HISTORIES[name], self.m) self.h.run(wait=wait) def test_init_sets_default_printer_width(self): assert self.p.width == len(SYSTEM_PRINTER_NAME) def test_add_process_updates_printer_width(self): self.m.add_process('interesting', 'ruby server.rb') assert self.p.width == len('interesting') def test_add_process_sets_name(self): proc = self.m.add_process('foo', 'ruby server.rb') assert proc.name == 'foo' def test_add_process_sets_cmd(self): proc = self.m.add_process('foo', 'ruby server.rb') assert proc.cmd == 'ruby server.rb' def test_add_process_sets_colour(self): proc = self.m.add_process('foo', 'ruby server.rb') assert proc.colour is not None def test_add_process_sets_unique_colours(self): p1 = self.m.add_process('foo', 'ruby server.rb') p2 = self.m.add_process('bar', 'python server.py') assert p1.colour != p2.colour def test_add_process_sets_quiet(self): proc = self.m.add_process('foo', 'ruby server.rb', quiet=True) assert proc.quiet def test_add_process_name_must_be_unique(self): self.m.add_process('foo', 'ruby server.rb') with pytest.raises(AssertionError): self.m.add_process('foo', 'another command') def test_add_process_sets_cwd(self): proc = self.m.add_process('foo', 'ruby server.rb', cwd='foo-dir') assert proc.cwd == 'foo-dir' def test_loop_with_empty_manager_returns_immediately(self): self.m.loop() def test_loop_calls_process_run(self): self.run_history('one') evts = self.h.find_events(type='run') assert len(evts) == 1 assert evts[0]['name'] == 'foo' assert evts[0]['events_passed'] def test_printer_receives_messages_in_correct_order(self): self.run_history('one') self.p.fetch_lines() assert self.p.lines_local[0].data == 'foo started (pid=123)\n' assert self.p.lines_local[1].data == b'hello, world!\n' assert self.p.lines_local[2].data == 'foo stopped (rc=0)\n' def test_printer_receives_lines_multi_process(self): self.run_history('two') l1 = self.p.find_line(b'process one\n') l2 = self.p.find_line(b'process two\n') assert l1.name == 'foo' assert l2.name == 'bar' def test_returncode_set_by_first_exiting_process(self): self.run_history('returncode') assert self.h.manager_returncode == 456 def test_printer_receives_lines_after_stop(self): self.run_history('output_after_stop') assert self.p.got_line(b'fishmongers\n') assert self.p.got_line(b'butchers\n')
def devserver(*, bootstrap: bool, workers: bool) -> None: "Starts all Snuba processes for local development." import os import sys from subprocess import call, list2cmdline from honcho.manager import Manager os.environ["PYTHONUNBUFFERED"] = "1" if bootstrap: cmd = ["snuba", "bootstrap", "--force", "--no-migrate"] if not workers: cmd.append("--no-kafka") returncode = call(cmd) if returncode > 0: sys.exit(returncode) # Run migrations returncode = call(["snuba", "migrations", "migrate", "--force"]) if returncode > 0: sys.exit(returncode) daemons = [("api", ["snuba", "api"])] if not workers: os.execvp(daemons[0][1][0], daemons[0][1]) daemons += [ ( "transaction-consumer", [ "snuba", "consumer", "--auto-offset-reset=latest", "--log-level=debug", "--storage=transactions", "--consumer-group=transactions_group", "--commit-log-topic=snuba-commit-log", ], ), ( "sessions-consumer", [ "snuba", "consumer", "--auto-offset-reset=latest", "--log-level=debug", "--storage=sessions_raw", "--consumer-group=sessions_group", ], ), ( "outcomes-consumer", [ "snuba", "consumer", "--auto-offset-reset=latest", "--log-level=debug", "--storage=outcomes_raw", "--consumer-group=outcomes_group", ], ), ( "consumer", [ "snuba", "consumer", "--auto-offset-reset=latest", "--log-level=debug", "--storage=errors", ], ), ( "replacer", [ "snuba", "replacer", "--auto-offset-reset=latest", "--log-level=debug", "--storage=errors", ], ), ( "subscriptions-consumer-events", [ "snuba", "subscriptions", "--auto-offset-reset=latest", "--log-level=debug", "--max-batch-size=1", "--consumer-group=snuba-events-subscriptions-consumers", "--dataset=events", "--commit-log-topic=snuba-commit-log", "--commit-log-group=snuba-consumers", "--delay-seconds=1", "--schedule-ttl=10", "--max-query-workers=1", ], ), ( "subscriptions-consumer-transactions", [ "snuba", "subscriptions", "--auto-offset-reset=latest", "--log-level=debug", "--max-batch-size=1", "--consumer-group=snuba-transactions-subscriptions-consumers", "--dataset=transactions", "--commit-log-topic=snuba-commit-log", "--commit-log-group=transactions_group", "--delay-seconds=1", "--schedule-ttl=10", "--max-query-workers=1", ], ), ( "cdc-consumer", [ "snuba", "multistorage-consumer", "--auto-offset-reset=latest", "--log-level=debug", "--storage=groupedmessages", "--storage=groupassignees", "--consumer-group=cdc_group", ], ), ] manager = Manager() for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, ) manager.loop() sys.exit(manager.returncode)
def devserver(reload, watchers, workers, bind): "Starts a lightweight web server for development." if ':' in bind: host, port = bind.split(':', 1) port = int(port) else: host = bind port = None import os from django.conf import settings from sentry.services.http import SentryHTTPServer uwsgi_overrides = { # Make sure we don't try and use uwsgi protocol 'protocol': 'http', # Make sure we reload really quickly for local dev in case it # doesn't want to shut down nicely on it's own, NO MERCY 'worker-reload-mercy': 2, } if reload: uwsgi_overrides['py-autoreload'] = 1 daemons = [] if watchers: daemons += settings.SENTRY_WATCHERS if workers: if settings.CELERY_ALWAYS_EAGER: raise click.ClickException( 'Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.' ) daemons += [ ('worker', ['sentry', 'celery', 'worker', '-c', '1', '-l', 'INFO']), ('beat', ['sentry', 'celery', 'beat', '-l', 'INFO']), ] # A better log-format for local dev when running through honcho, # but if there aren't any other daemons, we don't want to override. if daemons: uwsgi_overrides[ 'log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)' else: uwsgi_overrides[ 'log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)' server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides) # If we don't need any other daemons, just launch a normal uwsgi webserver # and avoid dealing with subprocesses if not daemons: return server.run() import sys from subprocess import list2cmdline from honcho.manager import Manager os.environ['PYTHONUNBUFFERED'] = 'true' # Make sure that the environment is prepared before honcho takes over # This sets all the appropriate uwsgi env vars, etc server.prepare_environment() daemons += [ ('server', ['sentry', 'run', 'web']), ] cwd = os.path.realpath( os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir)) manager = Manager() for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, cwd=cwd, ) manager.loop() sys.exit(manager.returncode)
def devserver(*, bootstrap: bool, workers: bool) -> None: "Starts all Snuba processes for local development." import os import sys from subprocess import list2cmdline, call from honcho.manager import Manager os.environ["PYTHONUNBUFFERED"] = "1" if bootstrap: cmd = ["snuba", "bootstrap", "--force"] if not workers: cmd.append("--no-kafka") returncode = call(cmd) if returncode > 0: sys.exit(returncode) daemons = [("api", ["snuba", "api"])] if not workers: os.execvp(daemons[0][1][0], daemons[0][1]) daemons += [ ( "transaction-consumer", [ "snuba", "consumer", "--auto-offset-reset=latest", "--log-level=debug", "--storage=transactions", "--consumer-group=transactions_group", ], ), ( "sessions-consumer", [ "snuba", "consumer", "--auto-offset-reset=latest", "--log-level=debug", "--storage=sessions_raw", "--consumer-group=sessions_group", ], ), ( "consumer", [ "snuba", "consumer", "--auto-offset-reset=latest", "--log-level=debug", "--storage=events", ], ), ( "replacer", [ "snuba", "replacer", "--auto-offset-reset=latest", "--log-level=debug", "--storage=events", ], ), ] manager = Manager() for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, ) manager.loop() sys.exit(manager.returncode)
class TestManager(object): @pytest.fixture(autouse=True) def printer(self): # noqa self.p = FakePrinter() self.m = Manager(printer=self.p) self.m._env = FakeEnv() def run_history(self, name, wait=True): self.h = Harness(HISTORIES[name], self.m) self.h.run(wait=wait) def test_init_sets_default_printer_width(self): assert self.p.width == len(SYSTEM_PRINTER_NAME) def test_add_process_updates_printer_width(self): self.m.add_process('interesting', 'ruby server.rb') assert self.p.width == len('interesting') def test_add_process_sets_name(self): proc = self.m.add_process('foo', 'ruby server.rb') assert proc.name == 'foo' def test_add_process_sets_cmd(self): proc = self.m.add_process('foo', 'ruby server.rb') assert proc.cmd == 'ruby server.rb' def test_add_process_sets_colour(self): proc = self.m.add_process('foo', 'ruby server.rb') assert proc.colour is not None def test_add_process_sets_unique_colours(self): p1 = self.m.add_process('foo', 'ruby server.rb') p2 = self.m.add_process('bar', 'python server.py') assert p1.colour != p2.colour def test_add_process_sets_quiet(self): proc = self.m.add_process('foo', 'ruby server.rb', quiet=True) assert proc.quiet def test_add_process_name_must_be_unique(self): self.m.add_process('foo', 'ruby server.rb') with pytest.raises(AssertionError): self.m.add_process('foo', 'another command') def test_add_process_sets_cwd(self): proc = self.m.add_process('foo', 'ruby server.rb', cwd='foo-dir') assert proc.cwd == 'foo-dir' def test_loop_with_empty_manager_returns_immediately(self): self.m.loop() def test_loop_calls_process_run(self): self.run_history('one') evts = self.h.find_events(type='run') assert len(evts) == 1 assert evts[0]['name'] == 'foo' assert evts[0]['events_passed'] def test_printer_receives_messages_in_correct_order(self): self.run_history('one') self.p.fetch_lines() assert self.p.lines_local[0].data == 'foo started (pid=123)\n' assert self.p.lines_local[1].data == b'hello, world!\n' assert self.p.lines_local[2].data == 'foo stopped (rc=0)\n' def test_printer_receives_lines_multi_process(self): self.run_history('two') l1 = self.p.find_line(b'process one\n') l2 = self.p.find_line(b'process two\n') assert l1.name == 'foo' assert l2.name == 'bar' def test_returncode_set_by_first_exiting_process(self): self.run_history('returncode') assert self.h.manager_returncode == 456 def test_printer_receives_lines_after_stop(self): self.run_history('output_after_stop') assert self.p.got_line(b'fishmongers\n') assert self.p.got_line(b'butchers\n')
def devserver(reload, watchers, workers, browser_reload, styleguide, prefix, environment, bind): "Starts a lightweight web server for development." if ':' in bind: host, port = bind.split(':', 1) port = int(port) else: host = bind port = None import os os.environ['SENTRY_ENVIRONMENT'] = environment from django.conf import settings from sentry import options from sentry.services.http import SentryHTTPServer url_prefix = options.get('system.url-prefix', '') parsed_url = urlparse(url_prefix) # Make sure we're trying to use a port that we can actually bind to needs_https = ( parsed_url.scheme == 'https' and (parsed_url.port or 443) > 1024 ) has_https = False if needs_https: from subprocess import check_output try: check_output(['which', 'https']) has_https = True except Exception: has_https = False from sentry.runner.initializer import show_big_error show_big_error( [ 'missing `https` on your `$PATH`, but https is needed', '`$ brew install mattrobenolt/stuff/https`', ] ) uwsgi_overrides = { # Make sure we don't try and use uwsgi protocol 'protocol': 'http', # Make sure we reload really quickly for local dev in case it # doesn't want to shut down nicely on it's own, NO MERCY 'worker-reload-mercy': 2, # We need stdin to support pdb in devserver 'honour-stdin': True, # accept ridiculously large files 'limit-post': 1 << 30, # do something with chunked 'http-chunked-input': True, 'thunder-lock': False, 'timeout': 600, 'harakiri': 600, } if reload: uwsgi_overrides['py-autoreload'] = 1 daemons = [] if watchers: daemons += settings.SENTRY_WATCHERS # When using browser_reload we proxy all requests through webpacks # devserver on the configured port. The backend is served on port+1 and is # proxied via the webpack configuration. if watchers and browser_reload: proxy_port = port port = port + 1 os.environ['SENTRY_WEBPACK_PROXY_PORT'] = '%s' % proxy_port os.environ['SENTRY_BACKEND_PORT'] = '%s' % port # Replace the webpack watcher with the drop-in webpack-dev-server webpack_config = next(w for w in daemons if w[0] == 'webpack')[1] webpack_config[0] = os.path.join( *os.path.split(webpack_config[0])[0:-1] + ('webpack-dev-server', ) ) daemons = [w for w in daemons if w[0] != 'webpack'] + [ ('webpack', webpack_config), ] if workers: if settings.CELERY_ALWAYS_EAGER: raise click.ClickException( 'Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.' ) daemons += [ ('worker', ['sentry', 'run', 'worker', '-c', '1', '--autoreload']), ('cron', ['sentry', 'run', 'cron', '--autoreload']), ] from sentry import eventstream if eventstream.requires_post_process_forwarder(): daemons += [ ('relay', ['sentry', 'run', 'post-process-forwarder', '--loglevel=debug', '--commit-batch-size=1']), ] if needs_https and has_https: https_port = six.text_type(parsed_url.port) https_host = parsed_url.hostname # Determine a random port for the backend http server import socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind((host, 0)) port = s.getsockname()[1] s.close() bind = '%s:%d' % (host, port) daemons += [ ('https', ['https', '-host', https_host, '-listen', host + ':' + https_port, bind]), ] # A better log-format for local dev when running through honcho, # but if there aren't any other daemons, we don't want to override. if daemons: uwsgi_overrides['log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)' else: uwsgi_overrides['log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)' server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides) # If we don't need any other daemons, just launch a normal uwsgi webserver # and avoid dealing with subprocesses if not daemons: return server.run() import sys from subprocess import list2cmdline from honcho.manager import Manager from honcho.printer import Printer os.environ['PYTHONUNBUFFERED'] = 'true' # Make sure that the environment is prepared before honcho takes over # This sets all the appropriate uwsgi env vars, etc server.prepare_environment() daemons += [ ('server', ['sentry', 'run', 'web']), ] if styleguide: daemons += [('storybook', ['yarn', 'storybook'])] cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir)) manager = Manager(Printer(prefix=prefix)) for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, cwd=cwd, ) manager.loop() sys.exit(manager.returncode)
def devserver(reload, watchers, workers, browser_reload, styleguide, prefix, environment, bind): "Starts a lightweight web server for development." if ':' in bind: host, port = bind.split(':', 1) port = int(port) else: host = bind port = None import os os.environ['SENTRY_ENVIRONMENT'] = environment from django.conf import settings from sentry import options from sentry.services.http import SentryHTTPServer url_prefix = options.get('system.url-prefix', '') parsed_url = urlparse(url_prefix) # Make sure we're trying to use a port that we can actually bind to needs_https = (parsed_url.scheme == 'https' and (parsed_url.port or 443) > 1024) has_https = False if needs_https: from subprocess import check_output try: check_output(['which', 'https']) has_https = True except Exception: has_https = False from sentry.runner.initializer import show_big_error show_big_error([ 'missing `https` on your `$PATH`, but https is needed', '`$ brew install mattrobenolt/stuff/https`', ]) uwsgi_overrides = { # Make sure we don't try and use uwsgi protocol 'protocol': 'http', # Make sure we reload really quickly for local dev in case it # doesn't want to shut down nicely on it's own, NO MERCY 'worker-reload-mercy': 2, # We need stdin to support pdb in devserver 'honour-stdin': True, # accept ridiculously large files 'limit-post': 1 << 30, # do something with chunked 'http-chunked-input': True, 'thunder-lock': False, 'timeout': 600, 'harakiri': 600, } if reload: uwsgi_overrides['py-autoreload'] = 1 daemons = [] if watchers: daemons += settings.SENTRY_WATCHERS # When using browser_reload we proxy all requests through webpacks # devserver on the configured port. The backend is served on port+1 and is # proxied via the webpack configuration. if watchers and browser_reload: proxy_port = port port = port + 1 os.environ['SENTRY_WEBPACK_PROXY_PORT'] = '%s' % proxy_port os.environ['SENTRY_BACKEND_PORT'] = '%s' % port # Replace the webpack watcher with the drop-in webpack-dev-server webpack_config = next(w for w in daemons if w[0] == 'webpack')[1] webpack_config[0] = os.path.join( *os.path.split(webpack_config[0])[0:-1] + ('webpack-dev-server', )) daemons = [w for w in daemons if w[0] != 'webpack'] + [ ('webpack', webpack_config), ] if workers: if settings.CELERY_ALWAYS_EAGER: raise click.ClickException( 'Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.' ) daemons += [ ('worker', ['sentry', 'run', 'worker', '-c', '1', '--autoreload']), ('cron', ['sentry', 'run', 'cron', '--autoreload']), ] from sentry import eventstream if eventstream.requires_post_process_forwarder(): daemons += [ ('relay', [ 'sentry', 'run', 'post-process-forwarder', '--loglevel=debug', '--commit-batch-size=1' ]), ] if needs_https and has_https: https_port = six.text_type(parsed_url.port) https_host = parsed_url.hostname # Determine a random port for the backend http server import socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind((host, 0)) port = s.getsockname()[1] s.close() bind = '%s:%d' % (host, port) daemons += [ ('https', [ 'https', '-host', https_host, '-listen', host + ':' + https_port, bind ]), ] # A better log-format for local dev when running through honcho, # but if there aren't any other daemons, we don't want to override. if daemons: uwsgi_overrides[ 'log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)' else: uwsgi_overrides[ 'log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)' server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides) # If we don't need any other daemons, just launch a normal uwsgi webserver # and avoid dealing with subprocesses if not daemons: return server.run() import sys from subprocess import list2cmdline from honcho.manager import Manager from honcho.printer import Printer os.environ['PYTHONUNBUFFERED'] = 'true' # Make sure that the environment is prepared before honcho takes over # This sets all the appropriate uwsgi env vars, etc server.prepare_environment() daemons += [ ('server', ['sentry', 'run', 'web']), ] if styleguide: daemons += [('storybook', ['yarn', 'storybook'])] cwd = os.path.realpath( os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir)) manager = Manager(Printer(prefix=prefix)) for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, cwd=cwd, ) manager.loop() sys.exit(manager.returncode)
def devserver(reload, watchers, workers, bind): "Starts a lightweight web server for development." if ':' in bind: host, port = bind.split(':', 1) port = int(port) else: host = bind port = None import os from django.conf import settings from sentry.services.http import SentryHTTPServer uwsgi_overrides = { # Make sure we don't try and use uwsgi protocol 'protocol': 'http', # Make sure we reload really quickly for local dev in case it # doesn't want to shut down nicely on it's own, NO MERCY 'worker-reload-mercy': 2, } if reload: uwsgi_overrides['py-autoreload'] = 1 daemons = [] if watchers: daemons += settings.SENTRY_WATCHERS if workers: if settings.CELERY_ALWAYS_EAGER: raise click.ClickException('Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.') daemons += [ ('worker', ['sentry', 'run', 'worker', '-c', '1', '-l', 'INFO', '--autoreload']), ('cron', ['sentry', 'run', 'cron', '-l', 'INFO', '--autoreload']), ] # A better log-format for local dev when running through honcho, # but if there aren't any other daemons, we don't want to override. if daemons: uwsgi_overrides['log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)' else: uwsgi_overrides['log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)' server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides) # If we don't need any other daemons, just launch a normal uwsgi webserver # and avoid dealing with subprocesses if not daemons: return server.run() import sys from subprocess import list2cmdline from honcho.manager import Manager os.environ['PYTHONUNBUFFERED'] = 'true' # Make sure that the environment is prepared before honcho takes over # This sets all the appropriate uwsgi env vars, etc server.prepare_environment() daemons += [ ('server', ['sentry', 'run', 'web']), ] cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir)) manager = Manager() for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, cwd=cwd, ) manager.loop() sys.exit(manager.returncode)
def devserver(reload, watchers, workers, bind): "Starts a lightweight web server for development." if ':' in bind: host, port = bind.split(':', 1) port = int(port) else: host = bind port = None import os from django.conf import settings from sentry.services.http import SentryHTTPServer uwsgi_overrides = { # Make sure we don't try and use uwsgi protocol 'protocol': 'http', # A better log-format for local dev 'log-format': '"%(method) %(uri) %(proto)" %(status) %(size) "%(referer)" "%(uagent)"' } if reload: uwsgi_overrides['py-autoreload'] = 1 server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides) daemons = [] if watchers: daemons += settings.SENTRY_WATCHERS if workers: if settings.CELERY_ALWAYS_EAGER: raise click.ClickException('Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.') daemons += [ ('worker', ['sentry', 'celery', 'worker', '-c', '1', '-l', 'INFO']), ('beat', ['sentry', 'celery', 'beat', '-l', 'INFO']), ] # If we don't need any other daemons, just launch a normal uwsgi webserver # and avoid dealing with subprocesses if not daemons: return server.run() import sys from subprocess import list2cmdline from honcho.manager import Manager os.environ['PYTHONUNBUFFERED'] = 'true' # Make sure that the environment is prepared before honcho takes over # This sets all the appropriate uwsgi env vars, etc server.prepare_environment() daemons += [ ('server', ['sentry', 'start']), ] # Change directory globally to affect all subprocesses. This is less than ideal, # but Honcho doesn't provide the ability to pass in a cwd to subprocesses yet. # See: https://github.com/nickstenning/honcho/pull/170 cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir)) os.chdir(cwd) manager = Manager() for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, env=os.environ.copy() ) manager.loop() sys.exit(manager.returncode)
def devserver(reload, watchers, workers, browser_reload, environment, bind): "Starts a lightweight web server for development." if ':' in bind: host, port = bind.split(':', 1) port = int(port) else: host = bind port = None import os os.environ['SENTRY_ENVIRONMENT'] = environment from django.conf import settings from sentry import options from sentry.services.http import SentryHTTPServer url_prefix = options.get('system.url-prefix', '') needs_https = url_prefix.startswith('https://') has_https = False if needs_https: from subprocess import check_output try: check_output(['which', 'https']) has_https = True except Exception: has_https = False from sentry.runner.initializer import show_big_error show_big_error([ 'missing `https` on your `$PATH`, but https is needed', '`$ brew install mattrobenolt/stuff/https`', ]) uwsgi_overrides = { # Make sure we don't try and use uwsgi protocol 'protocol': 'http', # Make sure we reload really quickly for local dev in case it # doesn't want to shut down nicely on it's own, NO MERCY 'worker-reload-mercy': 2, # We need stdin to support pdb in devserver 'honour-stdin': True, } if reload: uwsgi_overrides['py-autoreload'] = 1 daemons = [] if watchers: daemons += settings.SENTRY_WATCHERS if workers: if settings.CELERY_ALWAYS_EAGER: raise click.ClickException( 'Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.' ) daemons += [ ('worker', ['sentry', 'run', 'worker', '-c', '1', '--autoreload']), ('cron', ['sentry', 'run', 'cron', '--autoreload']), ] if needs_https and has_https: from six.moves.urllib.parse import urlparse parsed_url = urlparse(url_prefix) https_port = six.text_type(parsed_url.port or 443) https_host = parsed_url.hostname # Determine a random port for the backend http server import socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind((host, 0)) port = s.getsockname()[1] s.close() bind = '%s:%d' % (host, port) daemons += [ ('https', [ 'https', '-host', https_host, '-listen', host + ':' + https_port, bind ]), ] # A better log-format for local dev when running through honcho, # but if there aren't any other daemons, we don't want to override. if daemons: uwsgi_overrides[ 'log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)' else: uwsgi_overrides[ 'log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)' server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides) # If we don't need any other daemons, just launch a normal uwsgi webserver # and avoid dealing with subprocesses if not daemons: return server.run() import sys from subprocess import list2cmdline from honcho.manager import Manager os.environ['PYTHONUNBUFFERED'] = 'true' if browser_reload: os.environ['WEBPACK_LIVERELOAD'] = '1' # Make sure that the environment is prepared before honcho takes over # This sets all the appropriate uwsgi env vars, etc server.prepare_environment() daemons += [ ('server', ['sentry', 'run', 'web']), ] cwd = os.path.realpath( os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir)) manager = Manager() for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, cwd=cwd, ) manager.loop() sys.exit(manager.returncode)
def devserver(*, bootstrap: bool, workers: bool) -> None: "Starts all Snuba processes for local development." import os import sys from subprocess import list2cmdline, call from honcho.manager import Manager os.environ["PYTHONUNBUFFERED"] = "1" if bootstrap: cmd = ["snuba", "bootstrap", "--force"] if not workers: cmd.append("--no-kafka") returncode = call(cmd) if returncode > 0: sys.exit(returncode) daemons = [ ( "api", [ "uwsgi", "--master", "--manage-script-name", "--wsgi-file", "snuba/web/wsgi.py", "--http", "0.0.0.0:1218", "--http-keepalive", "--need-app", "--die-on-term", ], ), ] if not workers: os.execvp(daemons[0][1][0], daemons[0][1]) daemons += [ ( "transaction-consumer", [ "snuba", "consumer", "--auto-offset-reset=latest", "--log-level=debug", "--dataset=transactions", "--consumer-group=transactions_group", ], ), ( "consumer", [ "snuba", "consumer", "--auto-offset-reset=latest", "--log-level=debug" ], ), ( "replacer", [ "snuba", "replacer", "--auto-offset-reset=latest", "--log-level=debug" ], ), ] manager = Manager() for name, cmd in daemons: manager.add_process( name, list2cmdline(cmd), quiet=False, ) manager.loop() sys.exit(manager.returncode)
class TestManager(TestCase): def setUp(self): # noqa self.p = FakePrinter() self.m = Manager(printer=self.p) self.m._env = FakeEnv() def run_history(self, name, wait=True): self.h = Harness(HISTORIES[name], self.m) self.h.run(wait=wait) def test_init_sets_default_printer_width(self): self.assertEqual(len(SYSTEM_PRINTER_NAME), self.p.width) def test_add_process_updates_printer_width(self): self.m.add_process('interesting', 'ruby server.rb') self.assertEqual(len('interesting'), self.p.width) def test_add_process_sets_name(self): proc = self.m.add_process('foo', 'ruby server.rb') self.assertEqual('foo', proc.name) def test_add_process_sets_cmd(self): proc = self.m.add_process('foo', 'ruby server.rb') self.assertEqual('ruby server.rb', proc.cmd) def test_add_process_sets_colour(self): proc = self.m.add_process('foo', 'ruby server.rb') self.assertTrue(proc.colour is not None) def test_add_process_sets_unique_colours(self): p1 = self.m.add_process('foo', 'ruby server.rb') p2 = self.m.add_process('bar', 'python server.py') self.assertNotEqual(p1.colour, p2.colour) def test_add_process_sets_quiet(self): proc = self.m.add_process('foo', 'ruby server.rb', quiet=True) self.assertTrue(proc.quiet) def test_add_process_name_must_be_unique(self): self.m.add_process('foo', 'ruby server.rb') self.assertRaises(AssertionError, self.m.add_process, 'foo', 'another command') def test_loop_with_empty_manager_returns_immediately(self): self.m.loop() def test_loop_calls_process_run(self): self.run_history('one') evts = self.h.find_events(type='run') self.assertEqual(1, len(evts)) self.assertEqual('foo', evts[0]['name']) self.assertTrue(evts[0]['events_passed']) def test_printer_receives_messages_in_correct_order(self): self.run_history('one') self.p.fetch_lines() self.assertEqual('foo started (pid=123)\n', self.p.lines_local[0].data) self.assertEqual(b'hello, world!\n', self.p.lines_local[1].data) self.assertEqual('foo stopped (rc=0)\n', self.p.lines_local[2].data) def test_printer_receives_lines_multi_process(self): self.run_history('two') l1 = self.p.find_line(b'process one\n') l2 = self.p.find_line(b'process two\n') self.assertEqual('foo', l1.name) self.assertEqual('bar', l2.name) def test_returncode_set_by_first_exiting_process(self): self.run_history('returncode') self.assertEqual(456, self.h.manager_returncode) def test_printer_receives_lines_after_stop(self): self.run_history('output_after_stop') self.assertTrue(self.p.got_line(b'fishmongers\n')) self.assertTrue(self.p.got_line(b'butchers\n'))
class TestManager(TestCase): def setUp(self): # noqa self.p = FakePrinter() self.m = Manager(printer=self.p) self.m._env = FakeEnv() def run_history(self, name, wait=True): self.h = Harness(HISTORIES[name], self.m) self.h.run(wait=wait) def test_init_sets_default_printer_width(self): self.assertEqual(len(SYSTEM_PRINTER_NAME), self.p.width) def test_add_process_updates_printer_width(self): self.m.add_process('interesting', 'ruby server.rb') self.assertEqual(len('interesting'), self.p.width) def test_add_process_sets_name(self): proc = self.m.add_process('foo', 'ruby server.rb') self.assertEqual('foo', proc.name) def test_add_process_sets_cmd(self): proc = self.m.add_process('foo', 'ruby server.rb') self.assertEqual('ruby server.rb', proc.cmd) def test_add_process_sets_colour(self): proc = self.m.add_process('foo', 'ruby server.rb') self.assertTrue(proc.colour is not None) def test_add_process_sets_unique_colours(self): p1 = self.m.add_process('foo', 'ruby server.rb') p2 = self.m.add_process('bar', 'python server.py') self.assertNotEqual(p1.colour, p2.colour) def test_add_process_sets_quiet(self): proc = self.m.add_process('foo', 'ruby server.rb', quiet=True) self.assertTrue(proc.quiet) def test_add_process_name_must_be_unique(self): self.m.add_process('foo', 'ruby server.rb') self.assertRaises(AssertionError, self.m.add_process, 'foo', 'another command') def test_loop_with_empty_manager_returns_immediately(self): self.m.loop() def test_loop_calls_process_run(self): self.run_history('one') evts = self.h.find_events(type='run') self.assertEqual(1, len(evts)) self.assertEqual('foo', evts[0]['name']) self.assertTrue(evts[0]['events_passed']) def test_printer_receives_messages_in_correct_order(self): self.run_history('one') self.p.fetch_lines() self.assertEqual('foo started (pid=123)\n', self.p.lines_local[0].data) self.assertEqual(b'hello, world!\n', self.p.lines_local[1].data) self.assertEqual('foo stopped (rc=0)\n', self.p.lines_local[2].data) def test_printer_receives_lines_multi_process(self): self.run_history('two') l1 = self.p.find_line(b'process one\n') l2 = self.p.find_line(b'process two\n') self.assertEqual('foo', l1.name) self.assertEqual('bar', l2.name) def test_returncode_set_by_first_exiting_process(self): self.run_history('returncode') self.assertEqual(456, self.h.manager_returncode) def test_printer_receives_lines_after_stop(self): self.run_history('output_after_stop') self.assertTrue(self.p.got_line(b'fishmongers\n')) self.assertTrue(self.p.got_line(b'butchers\n'))
def devserver(https, web, ws, worker, assets, beat): """ Run a development server. This command will start a development instance of h, consisting of a web application, Celery worker, and websocket server. It will also start a process which will watch and build the frontend assets. By default, the webserver will be accessible at: http://localhost:5000 You can also pass the `--https` flag, which will look for a TLS certificate and key in PEM format in the current directory, in files called: .tlscert.pem .tlskey.pem If you use this flag, the webserver will be accessible at: https://localhost:5000 If you wish this to be the default behaviour, you can set the USE_HTTPS environment variable. """ try: from honcho.manager import Manager except ImportError: raise click.ClickException( "cannot import honcho: did you run `pip install -r requirements-dev.in` yet?" ) os.environ["PYTHONUNBUFFERED"] = "true" if https: gunicorn_args = "--certfile=.tlscert.pem --keyfile=.tlskey.pem" os.environ["APP_URL"] = "https://localhost:5000" os.environ["WEBSOCKET_URL"] = "wss://localhost:5001/ws" else: gunicorn_args = "" os.environ["APP_URL"] = "http://localhost:5000" os.environ["WEBSOCKET_URL"] = "ws://localhost:5001/ws" m = Manager() if web: m.add_process( "web", "newrelic-admin run-program gunicorn --name web --reload --paste conf/development-app.ini %s" % gunicorn_args, ) if ws: m.add_process( "ws", "newrelic-admin run-program gunicorn --name websocket --reload --paste conf/development-websocket.ini %s" % gunicorn_args, ) if worker: m.add_process("worker", "hypothesis --dev celery worker -l INFO") if beat: m.add_process("beat", "hypothesis --dev celery beat") if assets: m.add_process("assets", "gulp watch") m.loop() sys.exit(m.returncode)
def devserver( reload, watchers, workers, experimental_spa, styleguide, prefix, pretty, environment, debug_server, bind, ): "Starts a lightweight web server for development." if bind is None: bind = "127.0.0.1:8000" if ":" in bind: host, port = bind.split(":", 1) port = int(port) else: host = bind port = None import os os.environ["SENTRY_ENVIRONMENT"] = environment # NODE_ENV *must* use production for any prod-like environment as third party libraries look # for this magic constant os.environ["NODE_ENV"] = "production" if environment.startswith( "prod") else environment from django.conf import settings from sentry import options from sentry.services.http import SentryHTTPServer url_prefix = options.get("system.url-prefix", "") parsed_url = urlparse(url_prefix) # Make sure we're trying to use a port that we can actually bind to needs_https = parsed_url.scheme == "https" and (parsed_url.port or 443) > 1024 has_https = False if needs_https: from subprocess import check_output try: check_output(["which", "https"]) has_https = True except Exception: has_https = False from sentry.runner.initializer import show_big_error show_big_error([ "missing `https` on your `$PATH`, but https is needed", "`$ brew install mattrobenolt/stuff/https`", ]) uwsgi_overrides = { "http-keepalive": True, # Make sure we reload really quickly for local dev in case it # doesn't want to shut down nicely on it's own, NO MERCY "worker-reload-mercy": 2, # We need stdin to support pdb in devserver "honour-stdin": True, # accept ridiculously large files "limit-post": 1 << 30, # do something with chunked "http-chunked-input": True, "thunder-lock": False, "timeout": 600, "harakiri": 600, } if reload: uwsgi_overrides["py-autoreload"] = 1 daemons = [] if experimental_spa: os.environ["SENTRY_UI_DEV_ONLY"] = "1" if not watchers: click.secho( "Using experimental SPA mode without watchers enabled has no effect", err=True, fg="yellow", ) # We proxy all requests through webpacks devserver on the configured port. # The backend is served on port+1 and is proxied via the webpack # configuration. if watchers: daemons += settings.SENTRY_WATCHERS proxy_port = port port = port + 1 uwsgi_overrides["protocol"] = "http" os.environ["FORCE_WEBPACK_DEV_SERVER"] = "1" os.environ["SENTRY_WEBPACK_PROXY_PORT"] = "%s" % proxy_port os.environ["SENTRY_BACKEND_PORT"] = "%s" % port # webpack and/or typescript is causing memory issues os.environ["NODE_OPTIONS"] = ((os.environ.get("NODE_OPTIONS", "") + " --max-old-space-size=4096")).lstrip() # Replace the webpack watcher with the drop-in webpack-dev-server webpack_config = next(w for w in daemons if w[0] == "webpack")[1] webpack_config[0] = os.path.join( *os.path.split(webpack_config[0])[0:-1] + ("webpack-dev-server", )) daemons = [w for w in daemons if w[0] != "webpack" ] + [("webpack", webpack_config)] else: # If we are the bare http server, use the http option with uwsgi protocol # See https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html uwsgi_overrides.update({ # Make sure uWSGI spawns an HTTP server for us as we don't # have a proxy/load-balancer in front in dev mode. "http": "%s:%s" % (host, port), "protocol": "uwsgi", # This is needed to prevent https://git.io/fj7Lw "uwsgi-socket": None, }) if workers: if settings.CELERY_ALWAYS_EAGER: raise click.ClickException( "Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers." ) daemons += [_get_daemon("worker"), _get_daemon("cron")] from sentry import eventstream if eventstream.requires_post_process_forwarder(): daemons += [_get_daemon("post-process-forwarder")] if settings.SENTRY_DEV_PROCESS_SUBSCRIPTIONS: if not settings.SENTRY_EVENTSTREAM == "sentry.eventstream.kafka.KafkaEventStream": raise click.ClickException( "`SENTRY_DEV_PROCESS_SUBSCRIPTIONS` can only be used when " "`SENTRY_EVENTSTREAM=sentry.eventstream.kafka.KafkaEventStream`." ) for name, topic in settings.KAFKA_SUBSCRIPTION_RESULT_TOPICS.items( ): daemons += [ _get_daemon("subscription-consumer", "--topic", topic, suffix=name) ] if settings.SENTRY_USE_RELAY: daemons += [_get_daemon("ingest")] if needs_https and has_https: https_port = six.text_type(parsed_url.port) https_host = parsed_url.hostname # Determine a random port for the backend http server import socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind((host, 0)) port = s.getsockname()[1] s.close() bind = "%s:%d" % (host, port) daemons += [("https", [ "https", "-host", https_host, "-listen", host + ":" + https_port, bind ])] from sentry.runner.commands.devservices import _prepare_containers for name, container_options in _prepare_containers("sentry", silent=True).items(): if container_options.get("with_devserver", False): daemons += [(name, ["sentry", "devservices", "attach", "--fast", name])] # A better log-format for local dev when running through honcho, # but if there aren't any other daemons, we don't want to override. if daemons: uwsgi_overrides[ "log-format"] = "%(method) %(status) %(uri) %(proto) %(size)" else: uwsgi_overrides[ "log-format"] = "[%(ltime)] %(method) %(status) %(uri) %(proto) %(size)" server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides, debug=debug_server) # If we don't need any other daemons, just launch a normal uwsgi webserver # and avoid dealing with subprocesses if not daemons: return server.run() import sys from subprocess import list2cmdline from honcho.manager import Manager from honcho.printer import Printer os.environ["PYTHONUNBUFFERED"] = "true" if debug_server: threading.Thread(target=server.run).start() else: # Make sure that the environment is prepared before honcho takes over # This sets all the appropriate uwsgi env vars, etc server.prepare_environment() daemons += [_get_daemon("server")] if styleguide: daemons += [_get_daemon("storybook")] cwd = os.path.realpath( os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir)) honcho_printer = Printer(prefix=prefix) if pretty: from sentry.runner.formatting import monkeypatch_honcho_write honcho_printer.write = types.MethodType(monkeypatch_honcho_write, honcho_printer) manager = Manager(honcho_printer) for name, cmd in daemons: manager.add_process(name, list2cmdline(cmd), quiet=False, cwd=cwd) manager.loop() sys.exit(manager.returncode)
import sys from honcho.manager import Manager m = Manager() m.add_process('ls', 'python manage.py runserver') m.loop() sys.exit(m.returncode)