def main():
src_path = os.path.dirname(os.path.realpath(__file__))
base_path = os.path.normpath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
)
start = [
('api', ['python', 'runner.py', 'api'], src_path),
('web', ['python', 'runner.py', 'web'], src_path),
('play_server', ['python', 'runner.py', 'play_server'], src_path),
('worker', ['python', 'runner.py', 'worker'], src_path),
('npm', ['npm', 'run', 'start'], base_path),
]
if watchdog_installed:
start.append(
('play_scan_watch', ['python', 'runner.py', 'play_scan_watch'], src_path),
)
manager = Manager()
for name, cmd, cwd in start:
manager.add_process(
name,
list2cmdline(cmd),
quiet=False,
cwd=cwd,
)
manager.loop()
sys.exit(manager.returncode)
def command_start(args):
procfile_path = _procfile_path(args.app_root, args.procfile)
procfile = _procfile(procfile_path)
port = int(os.environ.get('PORT', args.port))
concurrency = _parse_concurrency(args.concurrency)
env = _read_env(procfile_path, args.env)
quiet = _parse_quiet(args.quiet)
if args.processes:
processes = compat.OrderedDict()
for name in args.processes:
try:
processes[name] = procfile.processes[name]
except KeyError:
raise CommandError("Process type '{0}' does not exist in Procfile".format(name))
else:
processes = procfile.processes
manager = Manager()
for p in environ.expand_processes(processes,
concurrency=concurrency,
env=env,
quiet=quiet,
port=port):
e = os.environ.copy()
e.update(p.env)
manager.add_process(p.name, p.cmd, quiet=p.quiet, env=e)
manager.loop()
sys.exit(manager.returncode)
def handle(self, *args, **options):
manager = Manager()
for i in range(options['num_workers']):
manager.add_process(
'worker{}'.format(i),
'otree runworker',
quiet=False,
env=self.get_env(options))
manager.loop()
sys.exit(manager.returncode)
def get_honcho_manager(self, options):
self.addr = self.get_addr(options['addr'])
self.port = self.get_port(options['port'])
manager = Manager()
daphne_cmd = 'daphne otree.asgi:channel_layer -b {} -p {}'.format(
self.addr,
self.port
)
print('Starting daphne server on {}:{}'.format(self.addr, self.port))
manager.add_process('daphne', daphne_cmd, env=self.get_env(options))
for i in range(3):
manager.add_process(
'worker{}'.format(i),
'otree runworker',
env=self.get_env(options))
if options['botworker']:
manager.add_process(
'botworker',
'otree botworker',
quiet=False,
env=self.get_env(options)
)
return manager
def get_honcho_manager(self, options):
manager = Manager()
manager.add_process(
'botworker',
'otree botworker',
quiet=False,
env=os.environ.copy()
)
manager.add_process(
'timeoutworkeronly',
'otree timeoutworkeronly',
quiet=False,
env=os.environ.copy()
)
return manager
def inner_run(self, *args, **options):
# Check a handler is registered for http reqs; if not, add default one
self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER]
self.channel_layer.router.check_default(
http_consumer=ViewConsumer(),
)
self.addr = self.get_addr(options['addr'])
self.port = self.get_port(options['port'])
# Run checks
self.stdout.write("Performing system checks...\n\n")
self.check(display_num_errors=True)
self.check_migrations()
# Print helpful text
quit_command = 'CTRL-BREAK' if sys.platform == 'win32' else 'CONTROL-C'
self.stdout.write((
"\n"
"otree-core version %(otree_version)s, "
"Django version %(django_version)s, "
"using settings %(settings)r\n"
"Starting web server at http://%(addr)s:%(port)s/\n"
# "Channel layer %(layer)s\n"
"Quit the server with %(quit_command)s.\n"
) % {
"django_version": self.get_version(),
"otree_version": otree.__version__,
"settings": settings.SETTINGS_MODULE,
"addr": '[%s]' % self.addr if self._raw_ipv6 else self.addr,
"port": self.port,
"quit_command": quit_command,
"layer": self.channel_layer,
})
self.stdout.flush()
# Launch server in 'main' thread. Signals are disabled as it's still
# actually a subthread under the autoreloader.
self.logger.debug("Daphne running, listening on %s:%s",
self.addr, self.port)
manager = Manager()
daphne_cmd = 'daphne otree.asgi:channel_layer -b {} -p {}'.format(
self.addr,
self.port
)
print(daphne_cmd)
manager.add_process('daphne', daphne_cmd, env=self.get_env(options))
for i in range(3):
manager.add_process(
'worker{}'.format(i),
'otree runworker',
env=self.get_env(options))
manager.loop()
sys.exit(manager.returncode)
def run(self):
from subprocess import list2cmdline
from honcho.manager import Manager as HonchoManager
os.environ['PYTHONUNBUFFERED'] = 'true'
daemons = [
('server', ['./manage.py', 'runsocketserver']),
('worker', ['./manage.py', 'celery']),
('cron', ['./manage.py', 'beat']),
('sass', ['./opinew.sh', 'sass_watcher']),
]
cwd = os.path.abspath(os.path.dirname(__file__))
honcho_manager = HonchoManager()
for name, cmd in daemons:
honcho_manager.add_process(
name, list2cmdline(cmd),
quiet=False, cwd=cwd,
)
honcho_manager.loop()
sys.exit(honcho_manager.returncode)
def handle(self, *args, **options):
manager = Manager()
manager.add_process(
'web',
'otree runweb',
quiet=False,
env=self.get_env(options))
manager.add_process(
'channels',
'otree runchannelsworker',
quiet=False,
env=self.get_env(options))
manager.add_process(
'celery',
'otree runceleryworker',
quiet=False,
env=self.get_env(options))
manager.loop()
sys.exit(manager.returncode)
def main():
src_path = os.path.dirname(os.path.realpath(__file__))
start = [
('api', ['python', 'runner.py', 'api'], src_path),
('web', ['python', 'runner.py', 'web'], src_path),
('play-server', ['python', 'runner.py', 'play-server'], src_path),
('worker', ['python', 'runner.py', 'worker'], src_path),
]
if watchdog_installed:
start.append(
('play-scan-watch', ['python', 'runner.py', 'play-scan-watch'
], src_path), )
manager = Manager()
for name, cmd, cwd in start:
manager.add_process(
name,
list2cmdline(cmd),
quiet=False,
cwd=cwd,
)
manager.loop()
def devserver(reload, watchers, workers, browser_reload, styleguide, prefix,
environment, bind):
"Starts a lightweight web server for development."
if ':' in bind:
host, port = bind.split(':', 1)
port = int(port)
else:
host = bind
port = None
import os
os.environ['SENTRY_ENVIRONMENT'] = environment
from django.conf import settings
from sentry import options
from sentry.services.http import SentryHTTPServer
url_prefix = options.get('system.url-prefix', '')
parsed_url = urlparse(url_prefix)
# Make sure we're trying to use a port that we can actually bind to
needs_https = (parsed_url.scheme == 'https'
and (parsed_url.port or 443) > 1024)
has_https = False
if needs_https:
from subprocess import check_output
try:
check_output(['which', 'https'])
has_https = True
except Exception:
has_https = False
from sentry.runner.initializer import show_big_error
show_big_error([
'missing `https` on your `$PATH`, but https is needed',
'`$ brew install mattrobenolt/stuff/https`',
])
uwsgi_overrides = {
# Make sure we don't try and use uwsgi protocol
'protocol': 'http',
# Make sure we reload really quickly for local dev in case it
# doesn't want to shut down nicely on it's own, NO MERCY
'worker-reload-mercy': 2,
# We need stdin to support pdb in devserver
'honour-stdin': True,
# accept ridiculously large files
'limit-post': 1 << 30,
# do something with chunked
'http-chunked-input': True,
'thunder-lock': False,
'timeout': 600,
'harakiri': 600,
}
if reload:
uwsgi_overrides['py-autoreload'] = 1
daemons = []
if watchers:
daemons += settings.SENTRY_WATCHERS
# When using browser_reload we proxy all requests through webpacks
# devserver on the configured port. The backend is served on port+1 and is
# proxied via the webpack configuration.
if watchers and browser_reload:
proxy_port = port
port = port + 1
os.environ['SENTRY_WEBPACK_PROXY_PORT'] = '%s' % proxy_port
os.environ['SENTRY_BACKEND_PORT'] = '%s' % port
# Replace the webpack watcher with the drop-in webpack-dev-server
webpack_config = next(w for w in daemons if w[0] == 'webpack')[1]
webpack_config[0] = os.path.join(
*os.path.split(webpack_config[0])[0:-1] + ('webpack-dev-server', ))
daemons = [w for w in daemons if w[0] != 'webpack'] + [
('webpack', webpack_config),
]
if workers:
if settings.CELERY_ALWAYS_EAGER:
raise click.ClickException(
'Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.'
)
daemons += [
('worker', ['sentry', 'run', 'worker', '-c', '1', '--autoreload']),
('cron', ['sentry', 'run', 'cron', '--autoreload']),
]
from sentry import eventstream
if eventstream.requires_post_process_forwarder():
daemons += [
('relay', [
'sentry', 'run', 'post-process-forwarder',
'--loglevel=debug', '--commit-batch-size=1'
]),
]
if needs_https and has_https:
https_port = six.text_type(parsed_url.port)
https_host = parsed_url.hostname
# Determine a random port for the backend http server
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host, 0))
port = s.getsockname()[1]
s.close()
bind = '%s:%d' % (host, port)
daemons += [
('https', [
'https', '-host', https_host, '-listen',
host + ':' + https_port, bind
]),
]
# A better log-format for local dev when running through honcho,
# but if there aren't any other daemons, we don't want to override.
if daemons:
uwsgi_overrides[
'log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
else:
uwsgi_overrides[
'log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'
server = SentryHTTPServer(host=host,
port=port,
workers=1,
extra_options=uwsgi_overrides)
# If we don't need any other daemons, just launch a normal uwsgi webserver
# and avoid dealing with subprocesses
if not daemons:
return server.run()
import sys
from subprocess import list2cmdline
from honcho.manager import Manager
from honcho.printer import Printer
os.environ['PYTHONUNBUFFERED'] = 'true'
# Make sure that the environment is prepared before honcho takes over
# This sets all the appropriate uwsgi env vars, etc
server.prepare_environment()
daemons += [
('server', ['sentry', 'run', 'web']),
]
if styleguide:
daemons += [('storybook', ['yarn', 'storybook'])]
cwd = os.path.realpath(
os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))
manager = Manager(Printer(prefix=prefix))
for name, cmd in daemons:
manager.add_process(
name,
list2cmdline(cmd),
quiet=False,
cwd=cwd,
)
manager.loop()
sys.exit(manager.returncode)
def devserver(reload, watchers, workers, bind):
"Starts a lightweight web server for development."
if ':' in bind:
host, port = bind.split(':', 1)
port = int(port)
else:
host = bind
port = None
import os
from django.conf import settings
from sentry.services.http import SentryHTTPServer
uwsgi_overrides = {
# Make sure we don't try and use uwsgi protocol
'protocol': 'http',
# Make sure we reload really quickly for local dev in case it
# doesn't want to shut down nicely on it's own, NO MERCY
'worker-reload-mercy': 2,
}
if reload:
uwsgi_overrides['py-autoreload'] = 1
daemons = []
if watchers:
daemons += settings.SENTRY_WATCHERS
if workers:
if settings.CELERY_ALWAYS_EAGER:
raise click.ClickException('Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.')
daemons += [
('worker', ['sentry', 'run', 'worker', '-c', '1', '-l', 'INFO', '--autoreload']),
('cron', ['sentry', 'run', 'cron', '-l', 'INFO', '--autoreload']),
]
# A better log-format for local dev when running through honcho,
# but if there aren't any other daemons, we don't want to override.
if daemons:
uwsgi_overrides['log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
else:
uwsgi_overrides['log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'
server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides)
# If we don't need any other daemons, just launch a normal uwsgi webserver
# and avoid dealing with subprocesses
if not daemons:
return server.run()
import sys
from subprocess import list2cmdline
from honcho.manager import Manager
os.environ['PYTHONUNBUFFERED'] = 'true'
# Make sure that the environment is prepared before honcho takes over
# This sets all the appropriate uwsgi env vars, etc
server.prepare_environment()
daemons += [
('server', ['sentry', 'run', 'web']),
]
cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))
manager = Manager()
for name, cmd in daemons:
manager.add_process(
name, list2cmdline(cmd),
quiet=False, cwd=cwd,
)
manager.loop()
sys.exit(manager.returncode)
def devserver(web, worker, sched, webpack):
"""
Run the flask development server, workers, and scheduler
"""
try:
from honcho.manager import Manager
except ImportError:
raise click.ClickException(
'cannot import honcho: did you run `pip install -e .` ?'
)
os.environ['PYTHONUNBUFFERED'] = 'true'
m = Manager()
if web:
m.add_process('web', 'python wsgi.py')
if worker:
m.add_process('worker', 'python worker.py')
if sched:
m.add_process('sched', 'python scheduler.py')
if webpack:
m.add_process('webpack', 'npm run start')
m.loop()
sys.exit(m.returncode)
def devserver(https, web, ws, worker, assets):
"""
Run a development server.
This command will start a development instance of h, consisting of a web
application, Celery worker, and websocket server. It will also start a
process which will watch and build the frontend assets.
By default, the webserver will be accessible at:
http://localhost:5000
You can also pass the `--https` flag, which will look for a TLS certificate
and key in PEM format in the current directory, in files called:
.tlscert.pem
.tlskey.pem
If you use this flag, the webserver will be accessible at:
https://localhost:5000
If you wish this to be the default behaviour, you can set the
USE_HTTPS environment variable.
"""
try:
from honcho.manager import Manager
except ImportError:
raise click.ClickException('cannot import honcho: did you run `pip install -r requirements-dev.in` yet?')
os.environ['PYTHONUNBUFFERED'] = 'true'
if https:
gunicorn_args = '--certfile=.tlscert.pem --keyfile=.tlskey.pem'
os.environ['APP_URL'] = 'https://localhost:5000'
os.environ['WEBSOCKET_URL'] = 'wss://localhost:5001/ws'
os.environ['ALLOWED_ORIGINS'] = ' '.join(
['https://localhost:5000', os.environ.get('ALLOWED_ORIGINS', '')])
else:
gunicorn_args = ''
os.environ['APP_URL'] = 'http://localhost:5000'
os.environ['WEBSOCKET_URL'] = 'ws://localhost:5001/ws'
m = Manager()
if web:
m.add_process('web',
'MODEL_CREATE_ALL=true '
'SEARCH_AUTOCONFIG=true '
'gunicorn --reload --paste conf/development-app.ini %s' % gunicorn_args)
if ws:
m.add_process('ws', 'gunicorn --reload --paste conf/development-websocket.ini %s' % gunicorn_args)
if worker:
m.add_process('worker', 'hypothesis --dev celery worker --autoreload')
if assets:
m.add_process('assets', 'gulp watch')
m.loop()
sys.exit(m.returncode)
class TestManager(object):
@pytest.fixture(autouse=True)
def printer(self): # noqa
self.p = FakePrinter()
self.m = Manager(printer=self.p)
self.m._env = FakeEnv()
def run_history(self, name, wait=True):
self.h = Harness(HISTORIES[name], self.m)
self.h.run(wait=wait)
def test_init_sets_default_printer_width(self):
assert self.p.width == len(SYSTEM_PRINTER_NAME)
def test_add_process_updates_printer_width(self):
self.m.add_process('interesting', 'ruby server.rb')
assert self.p.width == len('interesting')
def test_add_process_sets_name(self):
proc = self.m.add_process('foo', 'ruby server.rb')
assert proc.name == 'foo'
def test_add_process_sets_cmd(self):
proc = self.m.add_process('foo', 'ruby server.rb')
assert proc.cmd == 'ruby server.rb'
def test_add_process_sets_colour(self):
proc = self.m.add_process('foo', 'ruby server.rb')
assert proc.colour is not None
def test_add_process_sets_unique_colours(self):
p1 = self.m.add_process('foo', 'ruby server.rb')
p2 = self.m.add_process('bar', 'python server.py')
assert p1.colour != p2.colour
def test_add_process_sets_quiet(self):
proc = self.m.add_process('foo', 'ruby server.rb', quiet=True)
assert proc.quiet
def test_add_process_name_must_be_unique(self):
self.m.add_process('foo', 'ruby server.rb')
with pytest.raises(AssertionError):
self.m.add_process('foo', 'another command')
def test_add_process_sets_cwd(self):
proc = self.m.add_process('foo', 'ruby server.rb', cwd='foo-dir')
assert proc.cwd == 'foo-dir'
def test_loop_with_empty_manager_returns_immediately(self):
self.m.loop()
def test_loop_calls_process_run(self):
self.run_history('one')
evts = self.h.find_events(type='run')
assert len(evts) == 1
assert evts[0]['name'] == 'foo'
assert evts[0]['events_passed']
def test_printer_receives_messages_in_correct_order(self):
self.run_history('one')
self.p.fetch_lines()
assert self.p.lines_local[0].data == 'foo started (pid=123)\n'
assert self.p.lines_local[1].data == b'hello, world!\n'
assert self.p.lines_local[2].data == 'foo stopped (rc=0)\n'
def test_printer_receives_lines_multi_process(self):
self.run_history('two')
l1 = self.p.find_line(b'process one\n')
l2 = self.p.find_line(b'process two\n')
assert l1.name == 'foo'
assert l2.name == 'bar'
def test_returncode_set_by_first_exiting_process(self):
self.run_history('returncode')
assert self.h.manager_returncode == 456
def test_printer_receives_lines_after_stop(self):
self.run_history('output_after_stop')
assert self.p.got_line(b'fishmongers\n')
assert self.p.got_line(b'butchers\n')
def devserver(reload, watchers, workers, browser_reload, styleguide, prefix,
environment, bind, vscode_debug, search_index):
"Starts a lightweight web server for development."
if ':' in bind:
host, port = bind.split(':', 1)
port = int(port)
else:
host = bind
port = None
import os
os.environ['CLIMS_ENVIRONMENT'] = environment
from sentry.runner import configure
configure()
from django.conf import settings
from sentry import options
from sentry.services.http import SentryHTTPServer
# NOTE: We have to set this variable here rather than directly in the config file
# because the config file is imported before we're able to set the _ENVIRONMENT variable
settings.DEBUG = environment == 'development'
url_prefix = options.get('system.url-prefix', '')
parsed_url = urlparse(url_prefix)
# Make sure we're trying to use a port that we can actually bind to
needs_https = (parsed_url.scheme == 'https'
and (parsed_url.port or 443) > 1024)
has_https = False
if needs_https:
from subprocess import check_output
try:
check_output(['which', 'https'])
has_https = True
except Exception:
has_https = False
from sentry.runner.initializer import show_big_error
show_big_error([
'missing `https` on your `$PATH`, but https is needed',
'`$ brew install mattrobenolt/stuff/https`',
])
uwsgi_overrides = {
# Make sure we don't try and use uwsgi protocol
'protocol': 'http',
# Make sure we reload really quickly for local dev in case it
# doesn't want to shut down nicely on it's own, NO MERCY
'worker-reload-mercy': 2,
# We need stdin to support pdb in devserver
'honour-stdin': True,
# accept ridiculously large files
'limit-post': 1 << 30,
# do something with chunked
'http-chunked-input': True,
}
if reload:
uwsgi_overrides['py-autoreload'] = 1
daemons = []
if watchers and not browser_reload:
daemons += settings.SENTRY_WATCHERS
# For javascript dev, if browser reload and watchers, then:
# devserver listen on PORT + 1
# webpack dev server listen on PORT + 2
# proxy listen on PORT
if watchers and browser_reload:
new_port = port + 1
os.environ['WEBPACK_DEV_PROXY'] = '%s' % port
os.environ['WEBPACK_DEV_PORT'] = '%s' % (new_port + 1)
os.environ['SENTRY_DEVSERVER_PORT'] = '%s' % new_port
port = new_port
daemons += [('jsproxy', ['yarn', 'dev-proxy']),
('webpack', ['yarn', 'dev-server'])]
if workers:
if settings.CELERY_ALWAYS_EAGER:
raise click.ClickException(
'Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.'
)
daemons += [
('worker', ['lims', 'run', 'worker', '-c', '1', '--autoreload']),
('cron', ['lims', 'run', 'cron', '--autoreload']),
]
if needs_https and has_https:
https_port = six.text_type(parsed_url.port)
https_host = parsed_url.hostname
# Determine a random port for the backend http server
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host, 0))
port = s.getsockname()[1]
s.close()
bind = '%s:%d' % (host, port)
daemons += [
('https', [
'https', '-host', https_host, '-listen',
host + ':' + https_port, bind
]),
]
# A better log-format for local dev when running through honcho,
# but if there aren't any other daemons, we don't want to override.
if daemons:
uwsgi_overrides[
'log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
else:
uwsgi_overrides[
'log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'
server = SentryHTTPServer(host=host,
port=port,
workers=1,
extra_options=uwsgi_overrides)
# If using vscode_debug, currently we're only starting the builtin django server:
if vscode_debug:
from django.core.management import execute_from_command_line
execute_from_command_line(
['', 'runserver', '--noreload', '--nothreading'])
return
# If we don't need any other daemons, just launch a normal uwsgi webserver
# and avoid dealing with subprocesses
if not daemons:
return server.run()
import sys
from subprocess import list2cmdline
from honcho.manager import Manager
from honcho.printer import Printer
os.environ['PYTHONUNBUFFERED'] = 'true'
# Make sure that the environment is prepared before honcho takes over
# This sets all the appropriate uwsgi env vars, etc
server.prepare_environment()
daemons += [
('server', ['lims', 'run', 'web']),
]
if styleguide:
daemons += [('storybook', ['yarn', 'storybook'])]
if search_index and False:
daemons += [('search_index', ['lims', 'run', 'search_index'])]
cwd = os.path.realpath(
os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))
manager = Manager(Printer(prefix=prefix))
for name, cmd in daemons:
manager.add_process(
name,
list2cmdline(cmd),
quiet=False,
cwd=cwd,
)
manager.loop()
sys.exit(manager.returncode)
class TestManager(TestCase):
def setUp(self): # noqa
self.p = FakePrinter()
self.m = Manager(printer=self.p)
self.m._env = FakeEnv()
def run_history(self, name, wait=True):
self.h = Harness(HISTORIES[name], self.m)
self.h.run(wait=wait)
def test_init_sets_default_printer_width(self):
self.assertEqual(len(SYSTEM_PRINTER_NAME), self.p.width)
def test_add_process_updates_printer_width(self):
self.m.add_process('interesting', 'ruby server.rb')
self.assertEqual(len('interesting'), self.p.width)
def test_add_process_sets_name(self):
proc = self.m.add_process('foo', 'ruby server.rb')
self.assertEqual('foo', proc.name)
def test_add_process_sets_cmd(self):
proc = self.m.add_process('foo', 'ruby server.rb')
self.assertEqual('ruby server.rb', proc.cmd)
def test_add_process_sets_colour(self):
proc = self.m.add_process('foo', 'ruby server.rb')
self.assertTrue(proc.colour is not None)
def test_add_process_sets_unique_colours(self):
p1 = self.m.add_process('foo', 'ruby server.rb')
p2 = self.m.add_process('bar', 'python server.py')
self.assertNotEqual(p1.colour, p2.colour)
def test_add_process_sets_quiet(self):
proc = self.m.add_process('foo', 'ruby server.rb', quiet=True)
self.assertTrue(proc.quiet)
def test_add_process_name_must_be_unique(self):
self.m.add_process('foo', 'ruby server.rb')
self.assertRaises(AssertionError, self.m.add_process, 'foo',
'another command')
def test_loop_with_empty_manager_returns_immediately(self):
self.m.loop()
def test_loop_calls_process_run(self):
self.run_history('one')
evts = self.h.find_events(type='run')
self.assertEqual(1, len(evts))
self.assertEqual('foo', evts[0]['name'])
self.assertTrue(evts[0]['events_passed'])
def test_printer_receives_messages_in_correct_order(self):
self.run_history('one')
self.p.fetch_lines()
self.assertEqual('foo started (pid=123)\n', self.p.lines_local[0].data)
self.assertEqual(b'hello, world!\n', self.p.lines_local[1].data)
self.assertEqual('foo stopped (rc=0)\n', self.p.lines_local[2].data)
def test_printer_receives_lines_multi_process(self):
self.run_history('two')
l1 = self.p.find_line(b'process one\n')
l2 = self.p.find_line(b'process two\n')
self.assertEqual('foo', l1.name)
self.assertEqual('bar', l2.name)
def test_returncode_set_by_first_exiting_process(self):
self.run_history('returncode')
self.assertEqual(456, self.h.manager_returncode)
def test_printer_receives_lines_after_stop(self):
self.run_history('output_after_stop')
self.assertTrue(self.p.got_line(b'fishmongers\n'))
self.assertTrue(self.p.got_line(b'butchers\n'))
def devserver(reload, watchers, workers, experimental_spa, styleguide, prefix, environment, bind):
"Starts a lightweight web server for development."
if ":" in bind:
host, port = bind.split(":", 1)
port = int(port)
else:
host = bind
port = None
import os
os.environ["SENTRY_ENVIRONMENT"] = environment
from django.conf import settings
from sentry import options
from sentry.services.http import SentryHTTPServer
url_prefix = options.get("system.url-prefix", "")
parsed_url = urlparse(url_prefix)
# Make sure we're trying to use a port that we can actually bind to
needs_https = parsed_url.scheme == "https" and (parsed_url.port or 443) > 1024
has_https = False
if needs_https:
from subprocess import check_output
try:
check_output(["which", "https"])
has_https = True
except Exception:
has_https = False
from sentry.runner.initializer import show_big_error
show_big_error(
[
"missing `https` on your `$PATH`, but https is needed",
"`$ brew install mattrobenolt/stuff/https`",
]
)
uwsgi_overrides = {
# Make sure we don't try and use uwsgi protocol
"protocol": "http",
# Make sure we reload really quickly for local dev in case it
# doesn't want to shut down nicely on it's own, NO MERCY
"worker-reload-mercy": 2,
# We need stdin to support pdb in devserver
"honour-stdin": True,
# accept ridiculously large files
"limit-post": 1 << 30,
# do something with chunked
"http-chunked-input": True,
"thunder-lock": False,
"timeout": 600,
"harakiri": 600,
}
if reload:
uwsgi_overrides["py-autoreload"] = 1
daemons = []
if experimental_spa:
os.environ["SENTRY_EXPERIMENTAL_SPA"] = "1"
if not watchers:
click.secho(
"Using experimental SPA mode without watchers enabled has no effect",
err=True,
fg="yellow",
)
# We proxy all requests through webpacks devserver on the configured port.
# The backend is served on port+1 and is proxied via the webpack
# configuration.
if watchers:
daemons += settings.SENTRY_WATCHERS
proxy_port = port
port = port + 1
os.environ["SENTRY_WEBPACK_PROXY_PORT"] = "%s" % proxy_port
os.environ["SENTRY_BACKEND_PORT"] = "%s" % port
# Replace the webpack watcher with the drop-in webpack-dev-server
webpack_config = next(w for w in daemons if w[0] == "webpack")[1]
webpack_config[0] = os.path.join(
*os.path.split(webpack_config[0])[0:-1] + ("webpack-dev-server",)
)
daemons = [w for w in daemons if w[0] != "webpack"] + [("webpack", webpack_config)]
if workers:
if settings.CELERY_ALWAYS_EAGER:
raise click.ClickException(
"Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers."
)
daemons += [
("worker", ["sentry", "run", "worker", "-c", "1", "--autoreload"]),
("cron", ["sentry", "run", "cron", "--autoreload"]),
]
from sentry import eventstream
if eventstream.requires_post_process_forwarder():
daemons += [
(
"relay",
[
"sentry",
"run",
"post-process-forwarder",
"--loglevel=debug",
"--commit-batch-size=1",
],
)
]
if needs_https and has_https:
https_port = six.text_type(parsed_url.port)
https_host = parsed_url.hostname
# Determine a random port for the backend http server
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host, 0))
port = s.getsockname()[1]
s.close()
bind = "%s:%d" % (host, port)
daemons += [
("https", ["https", "-host", https_host, "-listen", host + ":" + https_port, bind])
]
# A better log-format for local dev when running through honcho,
# but if there aren't any other daemons, we don't want to override.
if daemons:
uwsgi_overrides["log-format"] = '"%(method) %(uri) %(proto)" %(status) %(size)'
else:
uwsgi_overrides["log-format"] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'
server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides)
# If we don't need any other daemons, just launch a normal uwsgi webserver
# and avoid dealing with subprocesses
if not daemons:
return server.run()
import sys
from subprocess import list2cmdline
from honcho.manager import Manager
from honcho.printer import Printer
os.environ["PYTHONUNBUFFERED"] = "true"
# Make sure that the environment is prepared before honcho takes over
# This sets all the appropriate uwsgi env vars, etc
server.prepare_environment()
daemons += [("server", ["sentry", "run", "web"])]
if styleguide:
daemons += [("storybook", ["./bin/yarn", "storybook"])]
cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))
manager = Manager(Printer(prefix=prefix))
for name, cmd in daemons:
manager.add_process(name, list2cmdline(cmd), quiet=False, cwd=cwd)
manager.loop()
sys.exit(manager.returncode)
def devserver(reload, watchers, workers, bind):
"Starts a lightweight web server for development."
if ':' in bind:
host, port = bind.split(':', 1)
port = int(port)
else:
host = bind
port = None
import os
from django.conf import settings
from sentry.services.http import SentryHTTPServer
uwsgi_overrides = {
# Make sure we don't try and use uwsgi protocol
'protocol': 'http',
# Make sure we reload really quickly for local dev in case it
# doesn't want to shut down nicely on it's own, NO MERCY
'worker-reload-mercy': 2,
}
if reload:
uwsgi_overrides['py-autoreload'] = 1
daemons = []
if watchers:
daemons += settings.SENTRY_WATCHERS
if workers:
if settings.CELERY_ALWAYS_EAGER:
raise click.ClickException(
'Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.'
)
daemons += [
('worker', ['sentry', 'celery', 'worker', '-c', '1', '-l',
'INFO']),
('beat', ['sentry', 'celery', 'beat', '-l', 'INFO']),
]
# A better log-format for local dev when running through honcho,
# but if there aren't any other daemons, we don't want to override.
if daemons:
uwsgi_overrides[
'log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
else:
uwsgi_overrides[
'log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'
server = SentryHTTPServer(host=host,
port=port,
workers=1,
extra_options=uwsgi_overrides)
# If we don't need any other daemons, just launch a normal uwsgi webserver
# and avoid dealing with subprocesses
if not daemons:
return server.run()
import sys
from subprocess import list2cmdline
from honcho.manager import Manager
os.environ['PYTHONUNBUFFERED'] = 'true'
# Make sure that the environment is prepared before honcho takes over
# This sets all the appropriate uwsgi env vars, etc
server.prepare_environment()
daemons += [
('server', ['sentry', 'run', 'web']),
]
cwd = os.path.realpath(
os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))
manager = Manager()
for name, cmd in daemons:
manager.add_process(
name,
list2cmdline(cmd),
quiet=False,
cwd=cwd,
)
manager.loop()
sys.exit(manager.returncode)
def devserver(*, bootstrap: bool, workers: bool) -> None:
"Starts all Snuba processes for local development."
import os
import sys
from subprocess import list2cmdline, call
from honcho.manager import Manager
os.environ["PYTHONUNBUFFERED"] = "1"
if bootstrap:
cmd = ["snuba", "bootstrap", "--force"]
if not workers:
cmd.append("--no-kafka")
returncode = call(cmd)
if returncode > 0:
sys.exit(returncode)
daemons = [("api", ["snuba", "api"])]
if not workers:
os.execvp(daemons[0][1][0], daemons[0][1])
daemons += [
(
"transaction-consumer",
[
"snuba",
"consumer",
"--auto-offset-reset=latest",
"--log-level=debug",
"--storage=transactions",
"--consumer-group=transactions_group",
],
),
(
"sessions-consumer",
[
"snuba",
"consumer",
"--auto-offset-reset=latest",
"--log-level=debug",
"--storage=sessions_raw",
"--consumer-group=sessions_group",
],
),
(
"consumer",
[
"snuba",
"consumer",
"--auto-offset-reset=latest",
"--log-level=debug",
"--storage=events",
],
),
(
"replacer",
[
"snuba",
"replacer",
"--auto-offset-reset=latest",
"--log-level=debug",
"--storage=events",
],
),
]
manager = Manager()
for name, cmd in daemons:
manager.add_process(
name,
list2cmdline(cmd),
quiet=False,
)
manager.loop()
sys.exit(manager.returncode)
def devserver(reload, watchers, workers, browser_reload, styleguide, prefix, environment, bind):
"Starts a lightweight web server for development."
if ':' in bind:
host, port = bind.split(':', 1)
port = int(port)
else:
host = bind
port = None
import os
os.environ['SENTRY_ENVIRONMENT'] = environment
from django.conf import settings
from sentry import options
from sentry.services.http import SentryHTTPServer
url_prefix = options.get('system.url-prefix', '')
parsed_url = urlparse(url_prefix)
# Make sure we're trying to use a port that we can actually bind to
needs_https = (
parsed_url.scheme == 'https' and
(parsed_url.port or 443) > 1024
)
has_https = False
if needs_https:
from subprocess import check_output
try:
check_output(['which', 'https'])
has_https = True
except Exception:
has_https = False
from sentry.runner.initializer import show_big_error
show_big_error(
[
'missing `https` on your `$PATH`, but https is needed',
'`$ brew install mattrobenolt/stuff/https`',
]
)
uwsgi_overrides = {
# Make sure we don't try and use uwsgi protocol
'protocol': 'http',
# Make sure we reload really quickly for local dev in case it
# doesn't want to shut down nicely on it's own, NO MERCY
'worker-reload-mercy': 2,
# We need stdin to support pdb in devserver
'honour-stdin': True,
# accept ridiculously large files
'limit-post': 1 << 30,
# do something with chunked
'http-chunked-input': True,
'thunder-lock': False,
'timeout': 600,
'harakiri': 600,
}
if reload:
uwsgi_overrides['py-autoreload'] = 1
daemons = []
if watchers:
daemons += settings.SENTRY_WATCHERS
# When using browser_reload we proxy all requests through webpacks
# devserver on the configured port. The backend is served on port+1 and is
# proxied via the webpack configuration.
if watchers and browser_reload:
proxy_port = port
port = port + 1
os.environ['SENTRY_WEBPACK_PROXY_PORT'] = '%s' % proxy_port
os.environ['SENTRY_BACKEND_PORT'] = '%s' % port
# Replace the webpack watcher with the drop-in webpack-dev-server
webpack_config = next(w for w in daemons if w[0] == 'webpack')[1]
webpack_config[0] = os.path.join(
*os.path.split(webpack_config[0])[0:-1] + ('webpack-dev-server', )
)
daemons = [w for w in daemons if w[0] != 'webpack'] + [
('webpack', webpack_config),
]
if workers:
if settings.CELERY_ALWAYS_EAGER:
raise click.ClickException(
'Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.'
)
daemons += [
('worker', ['sentry', 'run', 'worker', '-c', '1', '--autoreload']),
('cron', ['sentry', 'run', 'cron', '--autoreload']),
]
from sentry import eventstream
if eventstream.requires_post_process_forwarder():
daemons += [
('relay', ['sentry', 'run', 'post-process-forwarder', '--loglevel=debug', '--commit-batch-size=1']),
]
if needs_https and has_https:
https_port = six.text_type(parsed_url.port)
https_host = parsed_url.hostname
# Determine a random port for the backend http server
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host, 0))
port = s.getsockname()[1]
s.close()
bind = '%s:%d' % (host, port)
daemons += [
('https', ['https', '-host', https_host, '-listen', host + ':' + https_port, bind]),
]
# A better log-format for local dev when running through honcho,
# but if there aren't any other daemons, we don't want to override.
if daemons:
uwsgi_overrides['log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
else:
uwsgi_overrides['log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'
server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides)
# If we don't need any other daemons, just launch a normal uwsgi webserver
# and avoid dealing with subprocesses
if not daemons:
return server.run()
import sys
from subprocess import list2cmdline
from honcho.manager import Manager
from honcho.printer import Printer
os.environ['PYTHONUNBUFFERED'] = 'true'
# Make sure that the environment is prepared before honcho takes over
# This sets all the appropriate uwsgi env vars, etc
server.prepare_environment()
daemons += [
('server', ['sentry', 'run', 'web']),
]
if styleguide:
daemons += [('storybook', ['yarn', 'storybook'])]
cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))
manager = Manager(Printer(prefix=prefix))
for name, cmd in daemons:
manager.add_process(
name,
list2cmdline(cmd),
quiet=False,
cwd=cwd,
)
manager.loop()
sys.exit(manager.returncode)
class TestManager(object):
@pytest.fixture(autouse=True)
def printer(self): # noqa
self.p = FakePrinter()
self.m = Manager(printer=self.p)
self.m._env = FakeEnv()
def run_history(self, name, wait=True):
self.h = Harness(HISTORIES[name], self.m)
self.h.run(wait=wait)
def test_init_sets_default_printer_width(self):
assert self.p.width == len(SYSTEM_PRINTER_NAME)
def test_add_process_updates_printer_width(self):
self.m.add_process('interesting', 'ruby server.rb')
assert self.p.width == len('interesting')
def test_add_process_sets_name(self):
proc = self.m.add_process('foo', 'ruby server.rb')
assert proc.name == 'foo'
def test_add_process_sets_cmd(self):
proc = self.m.add_process('foo', 'ruby server.rb')
assert proc.cmd == 'ruby server.rb'
def test_add_process_sets_colour(self):
proc = self.m.add_process('foo', 'ruby server.rb')
assert proc.colour is not None
def test_add_process_sets_unique_colours(self):
p1 = self.m.add_process('foo', 'ruby server.rb')
p2 = self.m.add_process('bar', 'python server.py')
assert p1.colour != p2.colour
def test_add_process_sets_quiet(self):
proc = self.m.add_process('foo', 'ruby server.rb', quiet=True)
assert proc.quiet
def test_add_process_name_must_be_unique(self):
self.m.add_process('foo', 'ruby server.rb')
with pytest.raises(AssertionError):
self.m.add_process('foo', 'another command')
def test_add_process_sets_cwd(self):
proc = self.m.add_process('foo', 'ruby server.rb', cwd='foo-dir')
assert proc.cwd == 'foo-dir'
def test_loop_with_empty_manager_returns_immediately(self):
self.m.loop()
def test_loop_calls_process_run(self):
self.run_history('one')
evts = self.h.find_events(type='run')
assert len(evts) == 1
assert evts[0]['name'] == 'foo'
assert evts[0]['events_passed']
def test_printer_receives_messages_in_correct_order(self):
self.run_history('one')
self.p.fetch_lines()
assert self.p.lines_local[0].data == 'foo started (pid=123)\n'
assert self.p.lines_local[1].data == b'hello, world!\n'
assert self.p.lines_local[2].data == 'foo stopped (rc=0)\n'
def test_printer_receives_lines_multi_process(self):
self.run_history('two')
l1 = self.p.find_line(b'process one\n')
l2 = self.p.find_line(b'process two\n')
assert l1.name == 'foo'
assert l2.name == 'bar'
def test_returncode_set_by_first_exiting_process(self):
self.run_history('returncode')
assert self.h.manager_returncode == 456
def test_printer_receives_lines_after_stop(self):
self.run_history('output_after_stop')
assert self.p.got_line(b'fishmongers\n')
assert self.p.got_line(b'butchers\n')
class TestManager(TestCase):
def setUp(self): # noqa
self.p = FakePrinter()
self.m = Manager(printer=self.p)
self.m._env = FakeEnv()
def run_history(self, name, wait=True):
self.h = Harness(HISTORIES[name], self.m)
self.h.run(wait=wait)
def test_init_sets_default_printer_width(self):
self.assertEqual(len(SYSTEM_PRINTER_NAME), self.p.width)
def test_add_process_updates_printer_width(self):
self.m.add_process('interesting', 'ruby server.rb')
self.assertEqual(len('interesting'), self.p.width)
def test_add_process_sets_name(self):
proc = self.m.add_process('foo', 'ruby server.rb')
self.assertEqual('foo', proc.name)
def test_add_process_sets_cmd(self):
proc = self.m.add_process('foo', 'ruby server.rb')
self.assertEqual('ruby server.rb', proc.cmd)
def test_add_process_sets_colour(self):
proc = self.m.add_process('foo', 'ruby server.rb')
self.assertTrue(proc.colour is not None)
def test_add_process_sets_unique_colours(self):
p1 = self.m.add_process('foo', 'ruby server.rb')
p2 = self.m.add_process('bar', 'python server.py')
self.assertNotEqual(p1.colour, p2.colour)
def test_add_process_sets_quiet(self):
proc = self.m.add_process('foo', 'ruby server.rb', quiet=True)
self.assertTrue(proc.quiet)
def test_add_process_name_must_be_unique(self):
self.m.add_process('foo', 'ruby server.rb')
self.assertRaises(AssertionError, self.m.add_process, 'foo', 'another command')
def test_loop_with_empty_manager_returns_immediately(self):
self.m.loop()
def test_loop_calls_process_run(self):
self.run_history('one')
evts = self.h.find_events(type='run')
self.assertEqual(1, len(evts))
self.assertEqual('foo', evts[0]['name'])
self.assertTrue(evts[0]['events_passed'])
def test_printer_receives_messages_in_correct_order(self):
self.run_history('one')
self.p.fetch_lines()
self.assertEqual('foo started (pid=123)\n', self.p.lines_local[0].data)
self.assertEqual(b'hello, world!\n', self.p.lines_local[1].data)
self.assertEqual('foo stopped (rc=0)\n', self.p.lines_local[2].data)
def test_printer_receives_lines_multi_process(self):
self.run_history('two')
l1 = self.p.find_line(b'process one\n')
l2 = self.p.find_line(b'process two\n')
self.assertEqual('foo', l1.name)
self.assertEqual('bar', l2.name)
def test_returncode_set_by_first_exiting_process(self):
self.run_history('returncode')
self.assertEqual(456, self.h.manager_returncode)
def test_printer_receives_lines_after_stop(self):
self.run_history('output_after_stop')
self.assertTrue(self.p.got_line(b'fishmongers\n'))
self.assertTrue(self.p.got_line(b'butchers\n'))
def devserver(reload, watchers, workers, browser_reload, environment, bind):
"Starts a lightweight web server for development."
if ':' in bind:
host, port = bind.split(':', 1)
port = int(port)
else:
host = bind
port = None
import os
os.environ['SENTRY_ENVIRONMENT'] = environment
from django.conf import settings
from sentry import options
from sentry.services.http import SentryHTTPServer
url_prefix = options.get('system.url-prefix', '')
needs_https = url_prefix.startswith('https://')
has_https = False
if needs_https:
from subprocess import check_output
try:
check_output(['which', 'https'])
has_https = True
except Exception:
has_https = False
from sentry.runner.initializer import show_big_error
show_big_error([
'missing `https` on your `$PATH`, but https is needed',
'`$ brew install mattrobenolt/stuff/https`',
])
uwsgi_overrides = {
# Make sure we don't try and use uwsgi protocol
'protocol': 'http',
# Make sure we reload really quickly for local dev in case it
# doesn't want to shut down nicely on it's own, NO MERCY
'worker-reload-mercy': 2,
# We need stdin to support pdb in devserver
'honour-stdin': True,
}
if reload:
uwsgi_overrides['py-autoreload'] = 1
daemons = []
if watchers:
daemons += settings.SENTRY_WATCHERS
if workers:
if settings.CELERY_ALWAYS_EAGER:
raise click.ClickException(
'Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.'
)
daemons += [
('worker', ['sentry', 'run', 'worker', '-c', '1', '--autoreload']),
('cron', ['sentry', 'run', 'cron', '--autoreload']),
]
if needs_https and has_https:
from six.moves.urllib.parse import urlparse
parsed_url = urlparse(url_prefix)
https_port = six.text_type(parsed_url.port or 443)
https_host = parsed_url.hostname
# Determine a random port for the backend http server
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host, 0))
port = s.getsockname()[1]
s.close()
bind = '%s:%d' % (host, port)
daemons += [
('https', [
'https', '-host', https_host, '-listen',
host + ':' + https_port, bind
]),
]
# A better log-format for local dev when running through honcho,
# but if there aren't any other daemons, we don't want to override.
if daemons:
uwsgi_overrides[
'log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
else:
uwsgi_overrides[
'log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'
server = SentryHTTPServer(host=host,
port=port,
workers=1,
extra_options=uwsgi_overrides)
# If we don't need any other daemons, just launch a normal uwsgi webserver
# and avoid dealing with subprocesses
if not daemons:
return server.run()
import sys
from subprocess import list2cmdline
from honcho.manager import Manager
os.environ['PYTHONUNBUFFERED'] = 'true'
if browser_reload:
os.environ['WEBPACK_LIVERELOAD'] = '1'
# Make sure that the environment is prepared before honcho takes over
# This sets all the appropriate uwsgi env vars, etc
server.prepare_environment()
daemons += [
('server', ['sentry', 'run', 'web']),
]
cwd = os.path.realpath(
os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))
manager = Manager()
for name, cmd in daemons:
manager.add_process(
name,
list2cmdline(cmd),
quiet=False,
cwd=cwd,
)
manager.loop()
sys.exit(manager.returncode)
def devserver(*, bootstrap: bool, workers: bool) -> None:
"Starts all Snuba processes for local development."
import os
import sys
from subprocess import call, list2cmdline
from honcho.manager import Manager
os.environ["PYTHONUNBUFFERED"] = "1"
if bootstrap:
cmd = ["snuba", "bootstrap", "--force", "--no-migrate"]
if not workers:
cmd.append("--no-kafka")
returncode = call(cmd)
if returncode > 0:
sys.exit(returncode)
# Run migrations
returncode = call(["snuba", "migrations", "migrate", "--force"])
if returncode > 0:
sys.exit(returncode)
daemons = [("api", ["snuba", "api"])]
if not workers:
os.execvp(daemons[0][1][0], daemons[0][1])
daemons += [
(
"transaction-consumer",
[
"snuba",
"consumer",
"--auto-offset-reset=latest",
"--no-strict-offset-reset",
"--log-level=debug",
"--storage=transactions",
"--consumer-group=transactions_group",
"--commit-log-topic=snuba-commit-log",
],
),
(
"sessions-consumer",
[
"snuba",
"consumer",
"--auto-offset-reset=latest",
"--no-strict-offset-reset",
"--log-level=debug",
"--storage=sessions_raw",
"--consumer-group=sessions_group",
],
),
(
"outcomes-consumer",
[
"snuba",
"consumer",
"--auto-offset-reset=latest",
"--no-strict-offset-reset",
"--log-level=debug",
"--storage=outcomes_raw",
"--consumer-group=outcomes_group",
],
),
(
"consumer",
[
"snuba",
"consumer",
"--auto-offset-reset=latest",
"--no-strict-offset-reset",
"--log-level=debug",
"--storage=errors",
],
),
(
"replacer",
[
"snuba",
"replacer",
"--auto-offset-reset=latest",
"--no-strict-offset-reset",
"--log-level=debug",
"--storage=errors",
],
),
(
"cdc-consumer",
[
"snuba",
"multistorage-consumer",
"--auto-offset-reset=latest",
"--no-strict-offset-reset",
"--log-level=debug",
"--storage=groupedmessages",
"--storage=groupassignees",
"--consumer-group=cdc_group",
],
),
]
if settings.SEPARATE_SCHEDULER_EXECUTOR_SUBSCRIPTIONS_DEV:
daemons += [
(
"subscriptions-scheduler-events",
[
"snuba",
"subscriptions-scheduler",
"--entity=events",
"--consumer-group=snuba-events-subscriptions-scheduler",
"--followed-consumer-group=snuba-consumers",
"--auto-offset-reset=latest",
"--log-level=debug",
"--delay-seconds=1",
"--schedule-ttl=10",
],
),
(
"subscriptions-executor-events",
[
"snuba",
"subscriptions-executor",
"--dataset=events",
"--entity=events",
"--consumer-group=snuba-events-subscription-executor",
"--auto-offset-reset=latest",
],
),
(
"subscriptions-scheduler-transactions",
[
"snuba",
"subscriptions-scheduler",
"--entity=transactions",
"--consumer-group=snuba-transactions-subscriptions-scheduler",
"--followed-consumer-group=transactions_group",
"--auto-offset-reset=latest",
"--log-level=debug",
"--delay-seconds=1",
"--schedule-ttl=10",
],
),
(
"subscriptions-executor-transactions",
[
"snuba",
"subscriptions-executor",
"--dataset=transactions",
"--entity=transactions",
"--consumer-group=snuba-transactions-subscription-executor",
"--auto-offset-reset=latest",
],
),
]
else:
daemons += [
(
"subscriptions-scheduler-executor-events",
[
"snuba",
"subscriptions-scheduler-executor",
"--dataset=events",
"--entity=events",
"--consumer-group=snuba-events-subscriptions-scheduler-executor",
"--followed-consumer-group=snuba-consumers",
"--auto-offset-reset=latest",
"--no-strict-offset-reset",
"--log-level=debug",
"--delay-seconds=1",
"--schedule-ttl=10",
"--stale-threshold-seconds=900",
],
),
(
"subscriptions-scheduler-executor-transactions",
[
"snuba",
"subscriptions-scheduler-executor",
"--dataset=transactions",
"--entity=transactions",
"--consumer-group=snuba-transactions-subscriptions-scheduler-executor",
"--followed-consumer-group=transactions_group",
"--auto-offset-reset=latest",
"--no-strict-offset-reset",
"--log-level=debug",
"--delay-seconds=1",
"--schedule-ttl=10",
"--stale-threshold-seconds=900",
],
),
]
if settings.ENABLE_SENTRY_METRICS_DEV:
daemons += [
(
"metrics-consumer",
[
"snuba",
"consumer",
"--storage=metrics_raw",
"--auto-offset-reset=latest",
"--no-strict-offset-reset",
"--log-level=debug",
"--consumer-group=snuba-metrics-consumers",
],
),
]
if settings.ENABLE_METRICS_SUBSCRIPTIONS:
if settings.SEPARATE_SCHEDULER_EXECUTOR_SUBSCRIPTIONS_DEV:
daemons += [
(
"subscriptions-scheduler-metrics-counters",
[
"snuba",
"subscriptions-scheduler",
"--entity=metrics_counters",
"--consumer-group=snuba-metrics-subscriptions-scheduler",
"--followed-consumer-group=snuba-metrics-consumers",
"--auto-offset-reset=latest",
"--log-level=debug",
"--delay-seconds=1",
"--schedule-ttl=10",
],
),
(
"subscriptions-scheduler-metrics-sets",
[
"snuba",
"subscriptions-scheduler",
"--entity=metrics_sets",
"--consumer-group=snuba-metrics-subscriptions-scheduler",
"--followed-consumer-group=snuba-metrics-consumers",
"--auto-offset-reset=latest",
"--log-level=debug",
"--delay-seconds=1",
"--schedule-ttl=10",
],
),
(
"subscriptions-executor-metrics",
[
"snuba",
"subscriptions-executor",
"--dataset=metrics",
"--entity=metrics_counters",
"--entity=metrics_sets",
"--consumer-group=snuba-metrics-subscription-executor",
"--auto-offset-reset=latest",
],
),
]
else:
daemons += [
(
"subscriptions-scheduler-executor-metrics",
[
"snuba",
"subscriptions-scheduler-executor",
"--dataset=metrics",
"--entity=metrics_sets",
"--entity=metrics_counters",
"--consumer-group=snuba-metrics-subscriptions-scheduler-executor",
"--followed-consumer-group=snuba-metrics-consumers",
"--auto-offset-reset=latest",
"--no-strict-offset-reset",
"--log-level=debug",
"--delay-seconds=1",
"--schedule-ttl=10",
],
),
]
if settings.ENABLE_PROFILES_CONSUMER:
daemons += [
(
"profiles",
[
"snuba",
"consumer",
"--auto-offset-reset=latest",
"--no-strict-offset-reset",
"--log-level=debug",
"--storage=profiles",
],
),
]
if settings.ENABLE_REPLAYS_CONSUMER:
daemons += [
(
"replays-consumer",
[
"snuba",
"consumer",
"--auto-offset-reset=latest",
"--log-level=debug",
"--storage=replays",
],
),
]
manager = Manager()
for name, cmd in daemons:
manager.add_process(
name,
list2cmdline(cmd),
quiet=False,
)
manager.loop()
sys.exit(manager.returncode)
def printer(self): # noqa
self.p = FakePrinter()
self.m = Manager(printer=self.p)
self.m._env = FakeEnv()
def devserver(https, web, ws, worker, assets, beat):
"""
Run a development server.
This command will start a development instance of h, consisting of a web
application, Celery worker, and websocket server. It will also start a
process which will watch and build the frontend assets.
By default, the webserver will be accessible at:
http://localhost:5000
You can also pass the `--https` flag, which will look for a TLS certificate
and key in PEM format in the current directory, in files called:
.tlscert.pem
.tlskey.pem
If you use this flag, the webserver will be accessible at:
https://localhost:5000
If you wish this to be the default behaviour, you can set the
USE_HTTPS environment variable.
"""
try:
from honcho.manager import Manager
except ImportError:
raise click.ClickException(
'cannot import honcho: did you run `pip install -r requirements-dev.in` yet?'
)
os.environ['PYTHONUNBUFFERED'] = 'true'
if https:
gunicorn_args = '--certfile=.tlscert.pem --keyfile=.tlskey.pem'
os.environ['APP_URL'] = 'https://localhost:5000'
os.environ['WEBSOCKET_URL'] = 'wss://localhost:5001/ws'
os.environ['ALLOWED_ORIGINS'] = ' '.join(
['https://localhost:5000',
os.environ.get('ALLOWED_ORIGINS', '')])
else:
gunicorn_args = ''
os.environ['APP_URL'] = 'http://localhost:5000'
os.environ['WEBSOCKET_URL'] = 'ws://localhost:5001/ws'
m = Manager()
if web:
m.add_process(
'web', 'gunicorn --reload --paste conf/development-app.ini %s' %
gunicorn_args)
if ws:
m.add_process(
'ws',
'gunicorn --reload --paste conf/development-websocket.ini %s' %
gunicorn_args)
if worker:
m.add_process('worker', 'hypothesis --dev celery worker --autoreload')
if beat:
m.add_process('beat', 'hypothesis --dev celery beat')
if assets:
m.add_process('assets', 'gulp watch')
m.loop()
sys.exit(m.returncode)
def devserver(reload, watchers, workers, bind):
"Starts a lightweight web server for development."
if ':' in bind:
host, port = bind.split(':', 1)
port = int(port)
else:
host = bind
port = None
import os
from django.conf import settings
from sentry.services.http import SentryHTTPServer
uwsgi_overrides = {
# Make sure we don't try and use uwsgi protocol
'protocol': 'http',
# A better log-format for local dev
'log-format': '"%(method) %(uri) %(proto)" %(status) %(size) "%(referer)" "%(uagent)"'
}
if reload:
uwsgi_overrides['py-autoreload'] = 1
server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides)
daemons = []
if watchers:
daemons += settings.SENTRY_WATCHERS
if workers:
if settings.CELERY_ALWAYS_EAGER:
raise click.ClickException('Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.')
daemons += [
('worker', ['sentry', 'celery', 'worker', '-c', '1', '-l', 'INFO']),
('beat', ['sentry', 'celery', 'beat', '-l', 'INFO']),
]
# If we don't need any other daemons, just launch a normal uwsgi webserver
# and avoid dealing with subprocesses
if not daemons:
return server.run()
import sys
from subprocess import list2cmdline
from honcho.manager import Manager
os.environ['PYTHONUNBUFFERED'] = 'true'
# Make sure that the environment is prepared before honcho takes over
# This sets all the appropriate uwsgi env vars, etc
server.prepare_environment()
daemons += [
('server', ['sentry', 'start']),
]
# Change directory globally to affect all subprocesses. This is less than ideal,
# but Honcho doesn't provide the ability to pass in a cwd to subprocesses yet.
# See: https://github.com/nickstenning/honcho/pull/170
cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))
os.chdir(cwd)
manager = Manager()
for name, cmd in daemons:
manager.add_process(
name, list2cmdline(cmd),
quiet=False, env=os.environ.copy()
)
manager.loop()
sys.exit(manager.returncode)
def devserver(*, bootstrap: bool, workers: bool) -> None:
"Starts all Snuba processes for local development."
import os
import sys
from subprocess import call, list2cmdline
from honcho.manager import Manager
os.environ["PYTHONUNBUFFERED"] = "1"
if bootstrap:
cmd = ["snuba", "bootstrap", "--force", "--no-migrate"]
if not workers:
cmd.append("--no-kafka")
returncode = call(cmd)
if returncode > 0:
sys.exit(returncode)
# Run migrations
returncode = call(["snuba", "migrations", "migrate", "--force"])
if returncode > 0:
sys.exit(returncode)
daemons = [("api", ["snuba", "api"])]
if not workers:
os.execvp(daemons[0][1][0], daemons[0][1])
daemons += [
(
"transaction-consumer",
[
"snuba",
"consumer",
"--auto-offset-reset=latest",
"--log-level=debug",
"--storage=transactions",
"--consumer-group=transactions_group",
"--commit-log-topic=snuba-commit-log",
],
),
(
"sessions-consumer",
[
"snuba",
"consumer",
"--auto-offset-reset=latest",
"--log-level=debug",
"--storage=sessions_raw",
"--consumer-group=sessions_group",
],
),
(
"outcomes-consumer",
[
"snuba",
"consumer",
"--auto-offset-reset=latest",
"--log-level=debug",
"--storage=outcomes_raw",
"--consumer-group=outcomes_group",
],
),
(
"consumer",
[
"snuba",
"consumer",
"--auto-offset-reset=latest",
"--log-level=debug",
"--storage=errors",
],
),
(
"replacer",
[
"snuba",
"replacer",
"--auto-offset-reset=latest",
"--log-level=debug",
"--storage=errors",
],
),
(
"subscriptions-consumer-events",
[
"snuba",
"subscriptions",
"--auto-offset-reset=latest",
"--log-level=debug",
"--max-batch-size=1",
"--consumer-group=snuba-events-subscriptions-consumers",
"--dataset=events",
"--commit-log-topic=snuba-commit-log",
"--commit-log-group=snuba-consumers",
"--delay-seconds=1",
"--schedule-ttl=10",
"--max-query-workers=1",
],
),
(
"subscriptions-consumer-transactions",
[
"snuba",
"subscriptions",
"--auto-offset-reset=latest",
"--log-level=debug",
"--max-batch-size=1",
"--consumer-group=snuba-transactions-subscriptions-consumers",
"--dataset=transactions",
"--commit-log-topic=snuba-commit-log",
"--commit-log-group=transactions_group",
"--delay-seconds=1",
"--schedule-ttl=10",
"--max-query-workers=1",
],
),
(
"cdc-consumer",
[
"snuba",
"multistorage-consumer",
"--auto-offset-reset=latest",
"--log-level=debug",
"--storage=groupedmessages",
"--storage=groupassignees",
"--consumer-group=cdc_group",
],
),
]
manager = Manager()
for name, cmd in daemons:
manager.add_process(
name, list2cmdline(cmd), quiet=False,
)
manager.loop()
sys.exit(manager.returncode)
def devserver(
environment,
workers,
host,
port,
ngrok,
ngrok_domain,
vcs_server,
vcs_server_port,
pubsub,
pubsub_port,
watchdog,
):
os.environ.setdefault("FLASK_DEBUG", "1")
os.environ["NODE_ENV"] = environment
if vcs_server:
os.environ["VCS_SERVER_ENDPOINT"] = "http://localhost:{}".format(
vcs_server_port)
if pubsub:
os.environ["PUBSUB_ENDPOINT"] = "http://localhost:{}".format(
pubsub_port)
if ngrok:
root_url = "https://{}.ngrok.io".format(ngrok_domain)
os.environ["SSL"] = "1"
os.environ["SERVER_NAME"] = "{}.ngrok.io".format(ngrok_domain)
else:
root_url = "http://{}:{}".format(host, port)
click.echo("Launching Zeus on {}".format(root_url))
# TODO(dcramer): pass required attributes to 'run' directly instead
# of relying on FLASK_DEBUG
daemons = [
("web",
["zeus", "run", "--host={}".format(host), "--port={}".format(port)]),
(
"webpack",
[
"node_modules/.bin/webpack",
"--watch",
"--config=config/webpack.config.js",
],
),
]
if vcs_server:
daemons.append((
"vcs-server",
[
"zeus",
"vcs-server",
"--host={}".format(host),
"--port={}".format(vcs_server_port),
],
))
if pubsub:
daemons.append((
"pubsub",
[
"zeus",
"pubsub",
"--host={}".format(host),
"--port={}".format(pubsub_port),
],
))
if workers:
daemons.append(
("worker", ["zeus", "worker", "--cron", "--log-level=INFO"]))
if ngrok:
daemons.append((
"ngrok",
["ngrok", "http", "-subdomain={}".format(ngrok_domain),
str(port)],
))
cwd = os.path.realpath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
manager = Manager()
for name, cmd in daemons:
manager.add_process(name, list2cmdline(cmd), quiet=False, cwd=cwd)
if watchdog:
event_handler = LoggingEventHandler()
observer = Observer()
observer.schedule(
event_handler,
os.path.join(os.path.dirname(__file__), os.pardir),
recursive=True,
)
observer.start()
manager.loop()
sys.exit(manager.returncode)
def devserver(reload, watchers, workers, browser_reload, environment, bind):
"Starts a lightweight web server for development."
if ':' in bind:
host, port = bind.split(':', 1)
port = int(port)
else:
host = bind
port = None
import os
os.environ['SENTRY_ENVIRONMENT'] = environment
from django.conf import settings
from sentry import options
from sentry.services.http import SentryHTTPServer
url_prefix = options.get('system.url-prefix', '')
needs_https = url_prefix.startswith('https://')
has_https = False
if needs_https:
from subprocess import check_output
try:
check_output(['which', 'https'])
has_https = True
except Exception:
has_https = False
from sentry.runner.initializer import show_big_error
show_big_error([
'missing `https` on your `$PATH`, but https is needed',
'`$ brew install mattrobenolt/stuff/https`',
])
uwsgi_overrides = {
# Make sure we don't try and use uwsgi protocol
'protocol': 'http',
# Make sure we reload really quickly for local dev in case it
# doesn't want to shut down nicely on it's own, NO MERCY
'worker-reload-mercy': 2,
# We need stdin to support pdb in devserver
'honour-stdin': True,
}
if reload:
uwsgi_overrides['py-autoreload'] = 1
daemons = []
if watchers:
daemons += settings.SENTRY_WATCHERS
if workers:
if settings.CELERY_ALWAYS_EAGER:
raise click.ClickException('Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.')
daemons += [
('worker', ['sentry', 'run', 'worker', '-c', '1', '--autoreload']),
('cron', ['sentry', 'run', 'cron', '--autoreload']),
]
if needs_https and has_https:
from six.moves.urllib.parse import urlparse
parsed_url = urlparse(url_prefix)
https_port = six.text_type(parsed_url.port or 443)
https_host = parsed_url.hostname
# Determine a random port for the backend http server
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host, 0))
port = s.getsockname()[1]
s.close()
bind = '%s:%d' % (host, port)
daemons += [
('https', ['https', '-host', https_host, '-listen', host + ':' + https_port, bind]),
]
# A better log-format for local dev when running through honcho,
# but if there aren't any other daemons, we don't want to override.
if daemons:
uwsgi_overrides['log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
else:
uwsgi_overrides['log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'
server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides)
# If we don't need any other daemons, just launch a normal uwsgi webserver
# and avoid dealing with subprocesses
if not daemons:
return server.run()
import sys
from subprocess import list2cmdline
from honcho.manager import Manager
os.environ['PYTHONUNBUFFERED'] = 'true'
if browser_reload:
os.environ['WEBPACK_LIVERELOAD'] = '1'
# Make sure that the environment is prepared before honcho takes over
# This sets all the appropriate uwsgi env vars, etc
server.prepare_environment()
daemons += [
('server', ['sentry', 'run', 'web']),
]
cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))
manager = Manager()
for name, cmd in daemons:
manager.add_process(
name, list2cmdline(cmd),
quiet=False, cwd=cwd,
)
manager.loop()
sys.exit(manager.returncode)
def devserver(*, bootstrap: bool, workers: bool) -> None:
"Starts all Snuba processes for local development."
import os
import sys
from subprocess import list2cmdline, call
from honcho.manager import Manager
os.environ["PYTHONUNBUFFERED"] = "1"
if bootstrap:
cmd = ["snuba", "bootstrap", "--force"]
if not workers:
cmd.append("--no-kafka")
returncode = call(cmd)
if returncode > 0:
sys.exit(returncode)
daemons = [
(
"api",
[
"uwsgi",
"--master",
"--manage-script-name",
"--wsgi-file",
"snuba/web/wsgi.py",
"--http",
"0.0.0.0:1218",
"--http-keepalive",
"--need-app",
"--die-on-term",
],
),
]
if not workers:
os.execvp(daemons[0][1][0], daemons[0][1])
daemons += [
(
"transaction-consumer",
[
"snuba",
"consumer",
"--auto-offset-reset=latest",
"--log-level=debug",
"--dataset=transactions",
"--consumer-group=transactions_group",
],
),
(
"consumer",
[
"snuba", "consumer", "--auto-offset-reset=latest",
"--log-level=debug"
],
),
(
"replacer",
[
"snuba", "replacer", "--auto-offset-reset=latest",
"--log-level=debug"
],
),
]
manager = Manager()
for name, cmd in daemons:
manager.add_process(
name,
list2cmdline(cmd),
quiet=False,
)
manager.loop()
sys.exit(manager.returncode)
def setUp(self): # noqa
self.p = FakePrinter()
self.m = Manager(printer=self.p)
self.m._env = FakeEnv()
def devserver(https, web, ws, worker, assets, beat):
"""
Run a development server.
This command will start a development instance of h, consisting of a web
application, Celery worker, and websocket server. It will also start a
process which will watch and build the frontend assets.
By default, the webserver will be accessible at:
http://localhost:5000
You can also pass the `--https` flag, which will look for a TLS certificate
and key in PEM format in the current directory, in files called:
.tlscert.pem
.tlskey.pem
If you use this flag, the webserver will be accessible at:
https://localhost:5000
If you wish this to be the default behaviour, you can set the
USE_HTTPS environment variable.
"""
try:
from honcho.manager import Manager
except ImportError:
raise click.ClickException(
"cannot import honcho: did you run `pip install -r requirements-dev.in` yet?"
)
os.environ["PYTHONUNBUFFERED"] = "true"
if https:
gunicorn_args = "--certfile=.tlscert.pem --keyfile=.tlskey.pem"
os.environ["APP_URL"] = "https://localhost:5000"
os.environ["WEBSOCKET_URL"] = "wss://localhost:5001/ws"
else:
gunicorn_args = ""
os.environ["APP_URL"] = "http://localhost:5000"
os.environ["WEBSOCKET_URL"] = "ws://localhost:5001/ws"
m = Manager()
if web:
m.add_process(
"web",
"newrelic-admin run-program gunicorn --name web --reload --paste conf/development-app.ini %s"
% gunicorn_args,
)
if ws:
m.add_process(
"ws",
"newrelic-admin run-program gunicorn --name websocket --reload --paste conf/development-websocket.ini %s"
% gunicorn_args,
)
if worker:
m.add_process("worker", "hypothesis --dev celery worker -l INFO")
if beat:
m.add_process("beat", "hypothesis --dev celery beat")
if assets:
m.add_process("assets", "gulp watch")
m.loop()
sys.exit(m.returncode)
def devserver(
reload,
watchers,
workers,
experimental_spa,
styleguide,
prefix,
pretty,
environment,
debug_server,
bind,
):
"Starts a lightweight web server for development."
if bind is None:
bind = "127.0.0.1:8000"
if ":" in bind:
host, port = bind.split(":", 1)
port = int(port)
else:
host = bind
port = None
import os
os.environ["SENTRY_ENVIRONMENT"] = environment
# NODE_ENV *must* use production for any prod-like environment as third party libraries look
# for this magic constant
os.environ["NODE_ENV"] = "production" if environment.startswith(
"prod") else environment
from django.conf import settings
from sentry import options
from sentry.services.http import SentryHTTPServer
url_prefix = options.get("system.url-prefix", "")
parsed_url = urlparse(url_prefix)
# Make sure we're trying to use a port that we can actually bind to
needs_https = parsed_url.scheme == "https" and (parsed_url.port
or 443) > 1024
has_https = False
if needs_https:
from subprocess import check_output
try:
check_output(["which", "https"])
has_https = True
except Exception:
has_https = False
from sentry.runner.initializer import show_big_error
show_big_error([
"missing `https` on your `$PATH`, but https is needed",
"`$ brew install mattrobenolt/stuff/https`",
])
uwsgi_overrides = {
"http-keepalive": True,
# Make sure we reload really quickly for local dev in case it
# doesn't want to shut down nicely on it's own, NO MERCY
"worker-reload-mercy": 2,
# We need stdin to support pdb in devserver
"honour-stdin": True,
# accept ridiculously large files
"limit-post": 1 << 30,
# do something with chunked
"http-chunked-input": True,
"thunder-lock": False,
"timeout": 600,
"harakiri": 600,
}
if reload:
uwsgi_overrides["py-autoreload"] = 1
daemons = []
if experimental_spa:
os.environ["SENTRY_UI_DEV_ONLY"] = "1"
if not watchers:
click.secho(
"Using experimental SPA mode without watchers enabled has no effect",
err=True,
fg="yellow",
)
# We proxy all requests through webpacks devserver on the configured port.
# The backend is served on port+1 and is proxied via the webpack
# configuration.
if watchers:
daemons += settings.SENTRY_WATCHERS
proxy_port = port
port = port + 1
uwsgi_overrides["protocol"] = "http"
os.environ["FORCE_WEBPACK_DEV_SERVER"] = "1"
os.environ["SENTRY_WEBPACK_PROXY_PORT"] = "%s" % proxy_port
os.environ["SENTRY_BACKEND_PORT"] = "%s" % port
# webpack and/or typescript is causing memory issues
os.environ["NODE_OPTIONS"] = ((os.environ.get("NODE_OPTIONS", "") +
" --max-old-space-size=4096")).lstrip()
# Replace the webpack watcher with the drop-in webpack-dev-server
webpack_config = next(w for w in daemons if w[0] == "webpack")[1]
webpack_config[0] = os.path.join(
*os.path.split(webpack_config[0])[0:-1] + ("webpack-dev-server", ))
daemons = [w for w in daemons if w[0] != "webpack"
] + [("webpack", webpack_config)]
else:
# If we are the bare http server, use the http option with uwsgi protocol
# See https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
uwsgi_overrides.update({
# Make sure uWSGI spawns an HTTP server for us as we don't
# have a proxy/load-balancer in front in dev mode.
"http": "%s:%s" % (host, port),
"protocol": "uwsgi",
# This is needed to prevent https://git.io/fj7Lw
"uwsgi-socket": None,
})
if workers:
if settings.CELERY_ALWAYS_EAGER:
raise click.ClickException(
"Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers."
)
daemons += [_get_daemon("worker"), _get_daemon("cron")]
from sentry import eventstream
if eventstream.requires_post_process_forwarder():
daemons += [_get_daemon("post-process-forwarder")]
if settings.SENTRY_DEV_PROCESS_SUBSCRIPTIONS:
if not settings.SENTRY_EVENTSTREAM == "sentry.eventstream.kafka.KafkaEventStream":
raise click.ClickException(
"`SENTRY_DEV_PROCESS_SUBSCRIPTIONS` can only be used when "
"`SENTRY_EVENTSTREAM=sentry.eventstream.kafka.KafkaEventStream`."
)
for name, topic in settings.KAFKA_SUBSCRIPTION_RESULT_TOPICS.items(
):
daemons += [
_get_daemon("subscription-consumer",
"--topic",
topic,
suffix=name)
]
if settings.SENTRY_USE_RELAY:
daemons += [_get_daemon("ingest")]
if needs_https and has_https:
https_port = six.text_type(parsed_url.port)
https_host = parsed_url.hostname
# Determine a random port for the backend http server
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host, 0))
port = s.getsockname()[1]
s.close()
bind = "%s:%d" % (host, port)
daemons += [("https", [
"https", "-host", https_host, "-listen", host + ":" + https_port,
bind
])]
from sentry.runner.commands.devservices import _prepare_containers
for name, container_options in _prepare_containers("sentry",
silent=True).items():
if container_options.get("with_devserver", False):
daemons += [(name,
["sentry", "devservices", "attach", "--fast", name])]
# A better log-format for local dev when running through honcho,
# but if there aren't any other daemons, we don't want to override.
if daemons:
uwsgi_overrides[
"log-format"] = "%(method) %(status) %(uri) %(proto) %(size)"
else:
uwsgi_overrides[
"log-format"] = "[%(ltime)] %(method) %(status) %(uri) %(proto) %(size)"
server = SentryHTTPServer(host=host,
port=port,
workers=1,
extra_options=uwsgi_overrides,
debug=debug_server)
# If we don't need any other daemons, just launch a normal uwsgi webserver
# and avoid dealing with subprocesses
if not daemons:
return server.run()
import sys
from subprocess import list2cmdline
from honcho.manager import Manager
from honcho.printer import Printer
os.environ["PYTHONUNBUFFERED"] = "true"
if debug_server:
threading.Thread(target=server.run).start()
else:
# Make sure that the environment is prepared before honcho takes over
# This sets all the appropriate uwsgi env vars, etc
server.prepare_environment()
daemons += [_get_daemon("server")]
if styleguide:
daemons += [_get_daemon("storybook")]
cwd = os.path.realpath(
os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))
honcho_printer = Printer(prefix=prefix)
if pretty:
from sentry.runner.formatting import monkeypatch_honcho_write
honcho_printer.write = types.MethodType(monkeypatch_honcho_write,
honcho_printer)
manager = Manager(honcho_printer)
for name, cmd in daemons:
manager.add_process(name, list2cmdline(cmd), quiet=False, cwd=cwd)
manager.loop()
sys.exit(manager.returncode)
import sys
from honcho.manager import Manager
m = Manager()
m.add_process('ls', 'python manage.py runserver')
m.loop()
sys.exit(m.returncode)
