Esempio n. 1
0
def main():
    src_path = os.path.dirname(os.path.realpath(__file__))
    base_path = os.path.normpath(
        os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
    )

    start = [
        ('api', ['python', 'runner.py', 'api'], src_path),
        ('web', ['python', 'runner.py', 'web'], src_path),
        ('play_server', ['python', 'runner.py', 'play_server'], src_path),
        ('worker', ['python', 'runner.py', 'worker'], src_path),
        ('npm', ['npm', 'run', 'start'], base_path),
    ]
    if watchdog_installed:
        start.append(
            ('play_scan_watch', ['python', 'runner.py', 'play_scan_watch'], src_path),
        )

    manager = Manager()
    for name, cmd, cwd in start:
        manager.add_process(
            name, 
            list2cmdline(cmd),
            quiet=False, 
            cwd=cwd,
        )

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 2
0
def command_start(args):
    procfile_path = _procfile_path(args.app_root, args.procfile)
    procfile = _procfile(procfile_path)

    port = int(os.environ.get('PORT', args.port))
    concurrency = _parse_concurrency(args.concurrency)
    env = _read_env(procfile_path, args.env)
    quiet = _parse_quiet(args.quiet)

    if args.processes:
        processes = compat.OrderedDict()
        for name in args.processes:
            try:
                processes[name] = procfile.processes[name]
            except KeyError:
                raise CommandError("Process type '{0}' does not exist in Procfile".format(name))
    else:
        processes = procfile.processes

    manager = Manager()

    for p in environ.expand_processes(processes,
                                      concurrency=concurrency,
                                      env=env,
                                      quiet=quiet,
                                      port=port):
        e = os.environ.copy()
        e.update(p.env)
        manager.add_process(p.name, p.cmd, quiet=p.quiet, env=e)

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 3
0
    def handle(self, *args, **options):
        manager = Manager()

        for i in range(options['num_workers']):
            manager.add_process(
                'worker{}'.format(i),
                'otree runworker',
                quiet=False,
                env=self.get_env(options))

        manager.loop()
        sys.exit(manager.returncode)
Esempio n. 4
0
    def get_honcho_manager(self, options):
        self.addr = self.get_addr(options['addr'])
        self.port = self.get_port(options['port'])

        manager = Manager()

        daphne_cmd = 'daphne otree.asgi:channel_layer -b {} -p {}'.format(
            self.addr,
            self.port
        )

        print('Starting daphne server on {}:{}'.format(self.addr, self.port))

        manager.add_process('daphne', daphne_cmd, env=self.get_env(options))
        for i in range(3):
            manager.add_process(
                'worker{}'.format(i),
                'otree runworker',
                env=self.get_env(options))
        if options['botworker']:
            manager.add_process(
                'botworker',
                'otree botworker',
                quiet=False,
                env=self.get_env(options)
            )

        return manager
Esempio n. 5
0
    def get_honcho_manager(self, options):

        manager = Manager()


        manager.add_process(
            'botworker',
            'otree botworker',
            quiet=False,
            env=os.environ.copy()
        )
        manager.add_process(
            'timeoutworkeronly',
            'otree timeoutworkeronly',
            quiet=False,
            env=os.environ.copy()
        )

        return manager
    def inner_run(self, *args, **options):
        # Check a handler is registered for http reqs; if not, add default one
        self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER]
        self.channel_layer.router.check_default(
            http_consumer=ViewConsumer(),
        )

        self.addr = self.get_addr(options['addr'])
        self.port = self.get_port(options['port'])

        # Run checks
        self.stdout.write("Performing system checks...\n\n")
        self.check(display_num_errors=True)
        self.check_migrations()

        # Print helpful text
        quit_command = 'CTRL-BREAK' if sys.platform == 'win32' else 'CONTROL-C'
        self.stdout.write((
            "\n"
            "otree-core version %(otree_version)s, "
            "Django version %(django_version)s, "
            "using settings %(settings)r\n"
            "Starting web server at http://%(addr)s:%(port)s/\n"
            # "Channel layer %(layer)s\n"
            "Quit the server with %(quit_command)s.\n"
        ) % {
            "django_version": self.get_version(),
            "otree_version": otree.__version__,
            "settings": settings.SETTINGS_MODULE,
            "addr": '[%s]' % self.addr if self._raw_ipv6 else self.addr,
            "port": self.port,
            "quit_command": quit_command,
            "layer": self.channel_layer,
        })
        self.stdout.flush()

        # Launch server in 'main' thread. Signals are disabled as it's still
        # actually a subthread under the autoreloader.
        self.logger.debug("Daphne running, listening on %s:%s",
                          self.addr, self.port)

        manager = Manager()

        daphne_cmd = 'daphne otree.asgi:channel_layer -b {} -p {}'.format(
            self.addr,
            self.port
        )
        print(daphne_cmd)

        manager.add_process('daphne', daphne_cmd, env=self.get_env(options))
        for i in range(3):
            manager.add_process(
                'worker{}'.format(i),
                'otree runworker',
                env=self.get_env(options))
        manager.loop()

        sys.exit(manager.returncode)
Esempio n. 7
0
    def run(self):
        from subprocess import list2cmdline
        from honcho.manager import Manager as HonchoManager

        os.environ['PYTHONUNBUFFERED'] = 'true'

        daemons = [
            ('server', ['./manage.py', 'runsocketserver']),
            ('worker', ['./manage.py', 'celery']),
            ('cron', ['./manage.py', 'beat']),
            ('sass', ['./opinew.sh', 'sass_watcher']),
        ]

        cwd = os.path.abspath(os.path.dirname(__file__))
        honcho_manager = HonchoManager()
        for name, cmd in daemons:
            honcho_manager.add_process(
                name, list2cmdline(cmd),
                quiet=False, cwd=cwd,
            )

        honcho_manager.loop()
        sys.exit(honcho_manager.returncode)
Esempio n. 8
0
    def handle(self, *args, **options):
        manager = Manager()

        manager.add_process(
            'web',
            'otree runweb',
            quiet=False,
            env=self.get_env(options))
        manager.add_process(
            'channels',
            'otree runchannelsworker',
            quiet=False,
            env=self.get_env(options))
        manager.add_process(
            'celery',
            'otree runceleryworker',
            quiet=False,
            env=self.get_env(options))

        manager.loop()
        sys.exit(manager.returncode)
Esempio n. 9
0
def main():
    src_path = os.path.dirname(os.path.realpath(__file__))

    start = [
        ('api', ['python', 'runner.py', 'api'], src_path),
        ('web', ['python', 'runner.py', 'web'], src_path),
        ('play-server', ['python', 'runner.py', 'play-server'], src_path),
        ('worker', ['python', 'runner.py', 'worker'], src_path),
    ]
    if watchdog_installed:
        start.append(
            ('play-scan-watch', ['python', 'runner.py', 'play-scan-watch'
                                 ], src_path), )

    manager = Manager()
    for name, cmd, cwd in start:
        manager.add_process(
            name,
            list2cmdline(cmd),
            quiet=False,
            cwd=cwd,
        )

    manager.loop()
Esempio n. 10
0
def devserver(reload, watchers, workers, browser_reload, styleguide, prefix,
              environment, bind):
    "Starts a lightweight web server for development."
    if ':' in bind:
        host, port = bind.split(':', 1)
        port = int(port)
    else:
        host = bind
        port = None

    import os

    os.environ['SENTRY_ENVIRONMENT'] = environment

    from django.conf import settings
    from sentry import options
    from sentry.services.http import SentryHTTPServer

    url_prefix = options.get('system.url-prefix', '')
    parsed_url = urlparse(url_prefix)
    # Make sure we're trying to use a port that we can actually bind to
    needs_https = (parsed_url.scheme == 'https'
                   and (parsed_url.port or 443) > 1024)
    has_https = False

    if needs_https:
        from subprocess import check_output
        try:
            check_output(['which', 'https'])
            has_https = True
        except Exception:
            has_https = False
            from sentry.runner.initializer import show_big_error
            show_big_error([
                'missing `https` on your `$PATH`, but https is needed',
                '`$ brew install mattrobenolt/stuff/https`',
            ])

    uwsgi_overrides = {
        # Make sure we don't try and use uwsgi protocol
        'protocol': 'http',
        # Make sure we reload really quickly for local dev in case it
        # doesn't want to shut down nicely on it's own, NO MERCY
        'worker-reload-mercy': 2,
        # We need stdin to support pdb in devserver
        'honour-stdin': True,
        # accept ridiculously large files
        'limit-post': 1 << 30,
        # do something with chunked
        'http-chunked-input': True,
        'thunder-lock': False,
        'timeout': 600,
        'harakiri': 600,
    }

    if reload:
        uwsgi_overrides['py-autoreload'] = 1

    daemons = []

    if watchers:
        daemons += settings.SENTRY_WATCHERS

    # When using browser_reload we proxy all requests through webpacks
    # devserver on the configured port. The backend is served on port+1 and is
    # proxied via the webpack configuration.
    if watchers and browser_reload:
        proxy_port = port
        port = port + 1

        os.environ['SENTRY_WEBPACK_PROXY_PORT'] = '%s' % proxy_port
        os.environ['SENTRY_BACKEND_PORT'] = '%s' % port

        # Replace the webpack watcher with the drop-in webpack-dev-server
        webpack_config = next(w for w in daemons if w[0] == 'webpack')[1]
        webpack_config[0] = os.path.join(
            *os.path.split(webpack_config[0])[0:-1] + ('webpack-dev-server', ))

        daemons = [w for w in daemons if w[0] != 'webpack'] + [
            ('webpack', webpack_config),
        ]

    if workers:
        if settings.CELERY_ALWAYS_EAGER:
            raise click.ClickException(
                'Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.'
            )

        daemons += [
            ('worker', ['sentry', 'run', 'worker', '-c', '1', '--autoreload']),
            ('cron', ['sentry', 'run', 'cron', '--autoreload']),
        ]

        from sentry import eventstream
        if eventstream.requires_post_process_forwarder():
            daemons += [
                ('relay', [
                    'sentry', 'run', 'post-process-forwarder',
                    '--loglevel=debug', '--commit-batch-size=1'
                ]),
            ]

    if needs_https and has_https:
        https_port = six.text_type(parsed_url.port)
        https_host = parsed_url.hostname

        # Determine a random port for the backend http server
        import socket
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        s.bind((host, 0))
        port = s.getsockname()[1]
        s.close()
        bind = '%s:%d' % (host, port)

        daemons += [
            ('https', [
                'https', '-host', https_host, '-listen',
                host + ':' + https_port, bind
            ]),
        ]

    # A better log-format for local dev when running through honcho,
    # but if there aren't any other daemons, we don't want to override.
    if daemons:
        uwsgi_overrides[
            'log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
    else:
        uwsgi_overrides[
            'log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'

    server = SentryHTTPServer(host=host,
                              port=port,
                              workers=1,
                              extra_options=uwsgi_overrides)

    # If we don't need any other daemons, just launch a normal uwsgi webserver
    # and avoid dealing with subprocesses
    if not daemons:
        return server.run()

    import sys
    from subprocess import list2cmdline
    from honcho.manager import Manager
    from honcho.printer import Printer

    os.environ['PYTHONUNBUFFERED'] = 'true'

    # Make sure that the environment is prepared before honcho takes over
    # This sets all the appropriate uwsgi env vars, etc
    server.prepare_environment()
    daemons += [
        ('server', ['sentry', 'run', 'web']),
    ]

    if styleguide:
        daemons += [('storybook', ['yarn', 'storybook'])]

    cwd = os.path.realpath(
        os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))

    manager = Manager(Printer(prefix=prefix))
    for name, cmd in daemons:
        manager.add_process(
            name,
            list2cmdline(cmd),
            quiet=False,
            cwd=cwd,
        )

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 11
0
def devserver(reload, watchers, workers, bind):
    "Starts a lightweight web server for development."
    if ':' in bind:
        host, port = bind.split(':', 1)
        port = int(port)
    else:
        host = bind
        port = None

    import os
    from django.conf import settings
    from sentry.services.http import SentryHTTPServer

    uwsgi_overrides = {
        # Make sure we don't try and use uwsgi protocol
        'protocol': 'http',
        # Make sure we reload really quickly for local dev in case it
        # doesn't want to shut down nicely on it's own, NO MERCY
        'worker-reload-mercy': 2,
    }

    if reload:
        uwsgi_overrides['py-autoreload'] = 1

    daemons = []

    if watchers:
        daemons += settings.SENTRY_WATCHERS

    if workers:
        if settings.CELERY_ALWAYS_EAGER:
            raise click.ClickException('Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.')

        daemons += [
            ('worker', ['sentry', 'run', 'worker', '-c', '1', '-l', 'INFO', '--autoreload']),
            ('cron', ['sentry', 'run', 'cron', '-l', 'INFO', '--autoreload']),
        ]

    # A better log-format for local dev when running through honcho,
    # but if there aren't any other daemons, we don't want to override.
    if daemons:
        uwsgi_overrides['log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
    else:
        uwsgi_overrides['log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'

    server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides)

    # If we don't need any other daemons, just launch a normal uwsgi webserver
    # and avoid dealing with subprocesses
    if not daemons:
        return server.run()

    import sys
    from subprocess import list2cmdline
    from honcho.manager import Manager

    os.environ['PYTHONUNBUFFERED'] = 'true'

    # Make sure that the environment is prepared before honcho takes over
    # This sets all the appropriate uwsgi env vars, etc
    server.prepare_environment()
    daemons += [
        ('server', ['sentry', 'run', 'web']),
    ]

    cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))

    manager = Manager()
    for name, cmd in daemons:
        manager.add_process(
            name, list2cmdline(cmd),
            quiet=False, cwd=cwd,
        )

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 12
0
def devserver(web, worker, sched, webpack):
    """
    Run the flask development server, workers, and scheduler
    """
    try:
        from honcho.manager import Manager
    except ImportError:
        raise click.ClickException(
            'cannot import honcho: did you run `pip install -e .` ?'
        )

    os.environ['PYTHONUNBUFFERED'] = 'true'

    m = Manager()

    if web:
        m.add_process('web', 'python wsgi.py')

    if worker:
        m.add_process('worker', 'python worker.py')

    if sched:
        m.add_process('sched', 'python scheduler.py')

    if webpack:
        m.add_process('webpack', 'npm run start')

    m.loop()

    sys.exit(m.returncode)
Esempio n. 13
0
def devserver(https, web, ws, worker, assets):
    """
    Run a development server.

    This command will start a development instance of h, consisting of a web
    application, Celery worker, and websocket server. It will also start a
    process which will watch and build the frontend assets.

    By default, the webserver will be accessible at:

        http://localhost:5000

    You can also pass the `--https` flag, which will look for a TLS certificate
    and key in PEM format in the current directory, in files called:

        .tlscert.pem
        .tlskey.pem

    If you use this flag, the webserver will be accessible at:

        https://localhost:5000

    If you wish this to be the default behaviour, you can set the
    USE_HTTPS environment variable.
    """
    try:
        from honcho.manager import Manager
    except ImportError:
        raise click.ClickException('cannot import honcho: did you run `pip install -r requirements-dev.in` yet?')

    os.environ['PYTHONUNBUFFERED'] = 'true'
    if https:
        gunicorn_args = '--certfile=.tlscert.pem --keyfile=.tlskey.pem'
        os.environ['APP_URL'] = 'https://localhost:5000'
        os.environ['WEBSOCKET_URL'] = 'wss://localhost:5001/ws'
        os.environ['ALLOWED_ORIGINS'] = ' '.join(
            ['https://localhost:5000', os.environ.get('ALLOWED_ORIGINS', '')])
    else:
        gunicorn_args = ''
        os.environ['APP_URL'] = 'http://localhost:5000'
        os.environ['WEBSOCKET_URL'] = 'ws://localhost:5001/ws'

    m = Manager()
    if web:
        m.add_process('web',
                      'MODEL_CREATE_ALL=true '
                      'SEARCH_AUTOCONFIG=true '
                      'gunicorn --reload --paste conf/development-app.ini %s' % gunicorn_args)

    if ws:
        m.add_process('ws', 'gunicorn --reload --paste conf/development-websocket.ini %s' % gunicorn_args)

    if worker:
        m.add_process('worker', 'hypothesis --dev celery worker --autoreload')

    if assets:
        m.add_process('assets', 'gulp watch')

    m.loop()

    sys.exit(m.returncode)
Esempio n. 14
0
class TestManager(object):
    @pytest.fixture(autouse=True)
    def printer(self):  # noqa
        self.p = FakePrinter()
        self.m = Manager(printer=self.p)
        self.m._env = FakeEnv()

    def run_history(self, name, wait=True):
        self.h = Harness(HISTORIES[name], self.m)
        self.h.run(wait=wait)

    def test_init_sets_default_printer_width(self):
        assert self.p.width == len(SYSTEM_PRINTER_NAME)

    def test_add_process_updates_printer_width(self):
        self.m.add_process('interesting', 'ruby server.rb')
        assert self.p.width == len('interesting')

    def test_add_process_sets_name(self):
        proc = self.m.add_process('foo', 'ruby server.rb')
        assert proc.name == 'foo'

    def test_add_process_sets_cmd(self):
        proc = self.m.add_process('foo', 'ruby server.rb')
        assert proc.cmd == 'ruby server.rb'

    def test_add_process_sets_colour(self):
        proc = self.m.add_process('foo', 'ruby server.rb')
        assert proc.colour is not None

    def test_add_process_sets_unique_colours(self):
        p1 = self.m.add_process('foo', 'ruby server.rb')
        p2 = self.m.add_process('bar', 'python server.py')
        assert p1.colour != p2.colour

    def test_add_process_sets_quiet(self):
        proc = self.m.add_process('foo', 'ruby server.rb', quiet=True)
        assert proc.quiet

    def test_add_process_name_must_be_unique(self):
        self.m.add_process('foo', 'ruby server.rb')
        with pytest.raises(AssertionError):
            self.m.add_process('foo', 'another command')

    def test_add_process_sets_cwd(self):
        proc = self.m.add_process('foo', 'ruby server.rb', cwd='foo-dir')
        assert proc.cwd == 'foo-dir'

    def test_loop_with_empty_manager_returns_immediately(self):
        self.m.loop()

    def test_loop_calls_process_run(self):
        self.run_history('one')
        evts = self.h.find_events(type='run')
        assert len(evts) == 1
        assert evts[0]['name'] == 'foo'
        assert evts[0]['events_passed']

    def test_printer_receives_messages_in_correct_order(self):
        self.run_history('one')
        self.p.fetch_lines()
        assert self.p.lines_local[0].data == 'foo started (pid=123)\n'
        assert self.p.lines_local[1].data == b'hello, world!\n'
        assert self.p.lines_local[2].data == 'foo stopped (rc=0)\n'

    def test_printer_receives_lines_multi_process(self):
        self.run_history('two')
        l1 = self.p.find_line(b'process one\n')
        l2 = self.p.find_line(b'process two\n')
        assert l1.name == 'foo'
        assert l2.name == 'bar'

    def test_returncode_set_by_first_exiting_process(self):
        self.run_history('returncode')
        assert self.h.manager_returncode == 456

    def test_printer_receives_lines_after_stop(self):
        self.run_history('output_after_stop')
        assert self.p.got_line(b'fishmongers\n')
        assert self.p.got_line(b'butchers\n')
Esempio n. 15
0
def devserver(reload, watchers, workers, browser_reload, styleguide, prefix,
              environment, bind, vscode_debug, search_index):
    "Starts a lightweight web server for development."

    if ':' in bind:
        host, port = bind.split(':', 1)
        port = int(port)
    else:
        host = bind
        port = None

    import os

    os.environ['CLIMS_ENVIRONMENT'] = environment

    from sentry.runner import configure
    configure()

    from django.conf import settings
    from sentry import options
    from sentry.services.http import SentryHTTPServer

    # NOTE: We have to set this variable here rather than directly in the config file
    # because the config file is imported before we're able to set the _ENVIRONMENT variable
    settings.DEBUG = environment == 'development'

    url_prefix = options.get('system.url-prefix', '')
    parsed_url = urlparse(url_prefix)
    # Make sure we're trying to use a port that we can actually bind to
    needs_https = (parsed_url.scheme == 'https'
                   and (parsed_url.port or 443) > 1024)
    has_https = False

    if needs_https:
        from subprocess import check_output
        try:
            check_output(['which', 'https'])
            has_https = True
        except Exception:
            has_https = False
            from sentry.runner.initializer import show_big_error
            show_big_error([
                'missing `https` on your `$PATH`, but https is needed',
                '`$ brew install mattrobenolt/stuff/https`',
            ])

    uwsgi_overrides = {
        # Make sure we don't try and use uwsgi protocol
        'protocol': 'http',
        # Make sure we reload really quickly for local dev in case it
        # doesn't want to shut down nicely on it's own, NO MERCY
        'worker-reload-mercy': 2,
        # We need stdin to support pdb in devserver
        'honour-stdin': True,
        # accept ridiculously large files
        'limit-post': 1 << 30,
        # do something with chunked
        'http-chunked-input': True,
    }

    if reload:
        uwsgi_overrides['py-autoreload'] = 1

    daemons = []

    if watchers and not browser_reload:
        daemons += settings.SENTRY_WATCHERS

    # For javascript dev, if browser reload and watchers, then:
    # devserver listen on PORT + 1
    # webpack dev server listen on PORT + 2
    # proxy listen on PORT
    if watchers and browser_reload:
        new_port = port + 1
        os.environ['WEBPACK_DEV_PROXY'] = '%s' % port
        os.environ['WEBPACK_DEV_PORT'] = '%s' % (new_port + 1)
        os.environ['SENTRY_DEVSERVER_PORT'] = '%s' % new_port
        port = new_port

        daemons += [('jsproxy', ['yarn', 'dev-proxy']),
                    ('webpack', ['yarn', 'dev-server'])]

    if workers:
        if settings.CELERY_ALWAYS_EAGER:
            raise click.ClickException(
                'Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.'
            )

        daemons += [
            ('worker', ['lims', 'run', 'worker', '-c', '1', '--autoreload']),
            ('cron', ['lims', 'run', 'cron', '--autoreload']),
        ]

    if needs_https and has_https:
        https_port = six.text_type(parsed_url.port)
        https_host = parsed_url.hostname

        # Determine a random port for the backend http server
        import socket
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        s.bind((host, 0))
        port = s.getsockname()[1]
        s.close()
        bind = '%s:%d' % (host, port)

        daemons += [
            ('https', [
                'https', '-host', https_host, '-listen',
                host + ':' + https_port, bind
            ]),
        ]

    # A better log-format for local dev when running through honcho,
    # but if there aren't any other daemons, we don't want to override.
    if daemons:
        uwsgi_overrides[
            'log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
    else:
        uwsgi_overrides[
            'log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'

    server = SentryHTTPServer(host=host,
                              port=port,
                              workers=1,
                              extra_options=uwsgi_overrides)

    # If using vscode_debug, currently we're only starting the builtin django server:
    if vscode_debug:
        from django.core.management import execute_from_command_line
        execute_from_command_line(
            ['', 'runserver', '--noreload', '--nothreading'])
        return
    # If we don't need any other daemons, just launch a normal uwsgi webserver
    # and avoid dealing with subprocesses

    if not daemons:
        return server.run()

    import sys
    from subprocess import list2cmdline
    from honcho.manager import Manager
    from honcho.printer import Printer

    os.environ['PYTHONUNBUFFERED'] = 'true'

    # Make sure that the environment is prepared before honcho takes over
    # This sets all the appropriate uwsgi env vars, etc
    server.prepare_environment()
    daemons += [
        ('server', ['lims', 'run', 'web']),
    ]

    if styleguide:
        daemons += [('storybook', ['yarn', 'storybook'])]

    if search_index and False:
        daemons += [('search_index', ['lims', 'run', 'search_index'])]

    cwd = os.path.realpath(
        os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))

    manager = Manager(Printer(prefix=prefix))
    for name, cmd in daemons:
        manager.add_process(
            name,
            list2cmdline(cmd),
            quiet=False,
            cwd=cwd,
        )

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 16
0
class TestManager(TestCase):
    def setUp(self):  # noqa
        self.p = FakePrinter()
        self.m = Manager(printer=self.p)
        self.m._env = FakeEnv()

    def run_history(self, name, wait=True):
        self.h = Harness(HISTORIES[name], self.m)
        self.h.run(wait=wait)

    def test_init_sets_default_printer_width(self):
        self.assertEqual(len(SYSTEM_PRINTER_NAME), self.p.width)

    def test_add_process_updates_printer_width(self):
        self.m.add_process('interesting', 'ruby server.rb')
        self.assertEqual(len('interesting'), self.p.width)

    def test_add_process_sets_name(self):
        proc = self.m.add_process('foo', 'ruby server.rb')
        self.assertEqual('foo', proc.name)

    def test_add_process_sets_cmd(self):
        proc = self.m.add_process('foo', 'ruby server.rb')
        self.assertEqual('ruby server.rb', proc.cmd)

    def test_add_process_sets_colour(self):
        proc = self.m.add_process('foo', 'ruby server.rb')
        self.assertTrue(proc.colour is not None)

    def test_add_process_sets_unique_colours(self):
        p1 = self.m.add_process('foo', 'ruby server.rb')
        p2 = self.m.add_process('bar', 'python server.py')
        self.assertNotEqual(p1.colour, p2.colour)

    def test_add_process_sets_quiet(self):
        proc = self.m.add_process('foo', 'ruby server.rb', quiet=True)
        self.assertTrue(proc.quiet)

    def test_add_process_name_must_be_unique(self):
        self.m.add_process('foo', 'ruby server.rb')
        self.assertRaises(AssertionError, self.m.add_process, 'foo',
                          'another command')

    def test_loop_with_empty_manager_returns_immediately(self):
        self.m.loop()

    def test_loop_calls_process_run(self):
        self.run_history('one')
        evts = self.h.find_events(type='run')
        self.assertEqual(1, len(evts))
        self.assertEqual('foo', evts[0]['name'])
        self.assertTrue(evts[0]['events_passed'])

    def test_printer_receives_messages_in_correct_order(self):
        self.run_history('one')
        self.p.fetch_lines()
        self.assertEqual('foo started (pid=123)\n', self.p.lines_local[0].data)
        self.assertEqual(b'hello, world!\n', self.p.lines_local[1].data)
        self.assertEqual('foo stopped (rc=0)\n', self.p.lines_local[2].data)

    def test_printer_receives_lines_multi_process(self):
        self.run_history('two')
        l1 = self.p.find_line(b'process one\n')
        l2 = self.p.find_line(b'process two\n')
        self.assertEqual('foo', l1.name)
        self.assertEqual('bar', l2.name)

    def test_returncode_set_by_first_exiting_process(self):
        self.run_history('returncode')
        self.assertEqual(456, self.h.manager_returncode)

    def test_printer_receives_lines_after_stop(self):
        self.run_history('output_after_stop')
        self.assertTrue(self.p.got_line(b'fishmongers\n'))
        self.assertTrue(self.p.got_line(b'butchers\n'))
Esempio n. 17
0
def devserver(reload, watchers, workers, experimental_spa, styleguide, prefix, environment, bind):
    "Starts a lightweight web server for development."
    if ":" in bind:
        host, port = bind.split(":", 1)
        port = int(port)
    else:
        host = bind
        port = None

    import os

    os.environ["SENTRY_ENVIRONMENT"] = environment

    from django.conf import settings
    from sentry import options
    from sentry.services.http import SentryHTTPServer

    url_prefix = options.get("system.url-prefix", "")
    parsed_url = urlparse(url_prefix)
    # Make sure we're trying to use a port that we can actually bind to
    needs_https = parsed_url.scheme == "https" and (parsed_url.port or 443) > 1024
    has_https = False

    if needs_https:
        from subprocess import check_output

        try:
            check_output(["which", "https"])
            has_https = True
        except Exception:
            has_https = False
            from sentry.runner.initializer import show_big_error

            show_big_error(
                [
                    "missing `https` on your `$PATH`, but https is needed",
                    "`$ brew install mattrobenolt/stuff/https`",
                ]
            )

    uwsgi_overrides = {
        # Make sure we don't try and use uwsgi protocol
        "protocol": "http",
        # Make sure we reload really quickly for local dev in case it
        # doesn't want to shut down nicely on it's own, NO MERCY
        "worker-reload-mercy": 2,
        # We need stdin to support pdb in devserver
        "honour-stdin": True,
        # accept ridiculously large files
        "limit-post": 1 << 30,
        # do something with chunked
        "http-chunked-input": True,
        "thunder-lock": False,
        "timeout": 600,
        "harakiri": 600,
    }

    if reload:
        uwsgi_overrides["py-autoreload"] = 1

    daemons = []

    if experimental_spa:
        os.environ["SENTRY_EXPERIMENTAL_SPA"] = "1"
        if not watchers:
            click.secho(
                "Using experimental SPA mode without watchers enabled has no effect",
                err=True,
                fg="yellow",
            )

    # We proxy all requests through webpacks devserver on the configured port.
    # The backend is served on port+1 and is proxied via the webpack
    # configuration.
    if watchers:
        daemons += settings.SENTRY_WATCHERS

        proxy_port = port
        port = port + 1

        os.environ["SENTRY_WEBPACK_PROXY_PORT"] = "%s" % proxy_port
        os.environ["SENTRY_BACKEND_PORT"] = "%s" % port

        # Replace the webpack watcher with the drop-in webpack-dev-server
        webpack_config = next(w for w in daemons if w[0] == "webpack")[1]
        webpack_config[0] = os.path.join(
            *os.path.split(webpack_config[0])[0:-1] + ("webpack-dev-server",)
        )

        daemons = [w for w in daemons if w[0] != "webpack"] + [("webpack", webpack_config)]

    if workers:
        if settings.CELERY_ALWAYS_EAGER:
            raise click.ClickException(
                "Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers."
            )

        daemons += [
            ("worker", ["sentry", "run", "worker", "-c", "1", "--autoreload"]),
            ("cron", ["sentry", "run", "cron", "--autoreload"]),
        ]

        from sentry import eventstream

        if eventstream.requires_post_process_forwarder():
            daemons += [
                (
                    "relay",
                    [
                        "sentry",
                        "run",
                        "post-process-forwarder",
                        "--loglevel=debug",
                        "--commit-batch-size=1",
                    ],
                )
            ]

    if needs_https and has_https:
        https_port = six.text_type(parsed_url.port)
        https_host = parsed_url.hostname

        # Determine a random port for the backend http server
        import socket

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        s.bind((host, 0))
        port = s.getsockname()[1]
        s.close()
        bind = "%s:%d" % (host, port)

        daemons += [
            ("https", ["https", "-host", https_host, "-listen", host + ":" + https_port, bind])
        ]

    # A better log-format for local dev when running through honcho,
    # but if there aren't any other daemons, we don't want to override.
    if daemons:
        uwsgi_overrides["log-format"] = '"%(method) %(uri) %(proto)" %(status) %(size)'
    else:
        uwsgi_overrides["log-format"] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'

    server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides)

    # If we don't need any other daemons, just launch a normal uwsgi webserver
    # and avoid dealing with subprocesses
    if not daemons:
        return server.run()

    import sys
    from subprocess import list2cmdline
    from honcho.manager import Manager
    from honcho.printer import Printer

    os.environ["PYTHONUNBUFFERED"] = "true"

    # Make sure that the environment is prepared before honcho takes over
    # This sets all the appropriate uwsgi env vars, etc
    server.prepare_environment()
    daemons += [("server", ["sentry", "run", "web"])]

    if styleguide:
        daemons += [("storybook", ["./bin/yarn", "storybook"])]

    cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))

    manager = Manager(Printer(prefix=prefix))
    for name, cmd in daemons:
        manager.add_process(name, list2cmdline(cmd), quiet=False, cwd=cwd)

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 18
0
def devserver(reload, watchers, workers, bind):
    "Starts a lightweight web server for development."
    if ':' in bind:
        host, port = bind.split(':', 1)
        port = int(port)
    else:
        host = bind
        port = None

    import os
    from django.conf import settings
    from sentry.services.http import SentryHTTPServer

    uwsgi_overrides = {
        # Make sure we don't try and use uwsgi protocol
        'protocol': 'http',
        # Make sure we reload really quickly for local dev in case it
        # doesn't want to shut down nicely on it's own, NO MERCY
        'worker-reload-mercy': 2,
    }

    if reload:
        uwsgi_overrides['py-autoreload'] = 1

    daemons = []

    if watchers:
        daemons += settings.SENTRY_WATCHERS

    if workers:
        if settings.CELERY_ALWAYS_EAGER:
            raise click.ClickException(
                'Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.'
            )

        daemons += [
            ('worker', ['sentry', 'celery', 'worker', '-c', '1', '-l',
                        'INFO']),
            ('beat', ['sentry', 'celery', 'beat', '-l', 'INFO']),
        ]

    # A better log-format for local dev when running through honcho,
    # but if there aren't any other daemons, we don't want to override.
    if daemons:
        uwsgi_overrides[
            'log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
    else:
        uwsgi_overrides[
            'log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'

    server = SentryHTTPServer(host=host,
                              port=port,
                              workers=1,
                              extra_options=uwsgi_overrides)

    # If we don't need any other daemons, just launch a normal uwsgi webserver
    # and avoid dealing with subprocesses
    if not daemons:
        return server.run()

    import sys
    from subprocess import list2cmdline
    from honcho.manager import Manager

    os.environ['PYTHONUNBUFFERED'] = 'true'

    # Make sure that the environment is prepared before honcho takes over
    # This sets all the appropriate uwsgi env vars, etc
    server.prepare_environment()
    daemons += [
        ('server', ['sentry', 'run', 'web']),
    ]

    cwd = os.path.realpath(
        os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))

    manager = Manager()
    for name, cmd in daemons:
        manager.add_process(
            name,
            list2cmdline(cmd),
            quiet=False,
            cwd=cwd,
        )

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 19
0
def devserver(*, bootstrap: bool, workers: bool) -> None:
    "Starts all Snuba processes for local development."
    import os
    import sys
    from subprocess import list2cmdline, call
    from honcho.manager import Manager

    os.environ["PYTHONUNBUFFERED"] = "1"

    if bootstrap:
        cmd = ["snuba", "bootstrap", "--force"]
        if not workers:
            cmd.append("--no-kafka")
        returncode = call(cmd)
        if returncode > 0:
            sys.exit(returncode)

    daemons = [("api", ["snuba", "api"])]

    if not workers:
        os.execvp(daemons[0][1][0], daemons[0][1])

    daemons += [
        (
            "transaction-consumer",
            [
                "snuba",
                "consumer",
                "--auto-offset-reset=latest",
                "--log-level=debug",
                "--storage=transactions",
                "--consumer-group=transactions_group",
            ],
        ),
        (
            "sessions-consumer",
            [
                "snuba",
                "consumer",
                "--auto-offset-reset=latest",
                "--log-level=debug",
                "--storage=sessions_raw",
                "--consumer-group=sessions_group",
            ],
        ),
        (
            "consumer",
            [
                "snuba",
                "consumer",
                "--auto-offset-reset=latest",
                "--log-level=debug",
                "--storage=events",
            ],
        ),
        (
            "replacer",
            [
                "snuba",
                "replacer",
                "--auto-offset-reset=latest",
                "--log-level=debug",
                "--storage=events",
            ],
        ),
    ]

    manager = Manager()
    for name, cmd in daemons:
        manager.add_process(
            name,
            list2cmdline(cmd),
            quiet=False,
        )

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 20
0
def devserver(reload, watchers, workers, browser_reload, styleguide, prefix, environment, bind):
    "Starts a lightweight web server for development."
    if ':' in bind:
        host, port = bind.split(':', 1)
        port = int(port)
    else:
        host = bind
        port = None

    import os

    os.environ['SENTRY_ENVIRONMENT'] = environment

    from django.conf import settings
    from sentry import options
    from sentry.services.http import SentryHTTPServer

    url_prefix = options.get('system.url-prefix', '')
    parsed_url = urlparse(url_prefix)
    # Make sure we're trying to use a port that we can actually bind to
    needs_https = (
        parsed_url.scheme == 'https' and
        (parsed_url.port or 443) > 1024
    )
    has_https = False

    if needs_https:
        from subprocess import check_output
        try:
            check_output(['which', 'https'])
            has_https = True
        except Exception:
            has_https = False
            from sentry.runner.initializer import show_big_error
            show_big_error(
                [
                    'missing `https` on your `$PATH`, but https is needed',
                    '`$ brew install mattrobenolt/stuff/https`',
                ]
            )

    uwsgi_overrides = {
        # Make sure we don't try and use uwsgi protocol
        'protocol': 'http',
        # Make sure we reload really quickly for local dev in case it
        # doesn't want to shut down nicely on it's own, NO MERCY
        'worker-reload-mercy': 2,
        # We need stdin to support pdb in devserver
        'honour-stdin': True,
        # accept ridiculously large files
        'limit-post': 1 << 30,
        # do something with chunked
        'http-chunked-input': True,
        'thunder-lock': False,
        'timeout': 600,
        'harakiri': 600,
    }

    if reload:
        uwsgi_overrides['py-autoreload'] = 1

    daemons = []

    if watchers:
        daemons += settings.SENTRY_WATCHERS

    # When using browser_reload we proxy all requests through webpacks
    # devserver on the configured port. The backend is served on port+1 and is
    # proxied via the webpack configuration.
    if watchers and browser_reload:
        proxy_port = port
        port = port + 1

        os.environ['SENTRY_WEBPACK_PROXY_PORT'] = '%s' % proxy_port
        os.environ['SENTRY_BACKEND_PORT'] = '%s' % port

        # Replace the webpack watcher with the drop-in webpack-dev-server
        webpack_config = next(w for w in daemons if w[0] == 'webpack')[1]
        webpack_config[0] = os.path.join(
            *os.path.split(webpack_config[0])[0:-1] + ('webpack-dev-server', )
        )

        daemons = [w for w in daemons if w[0] != 'webpack'] + [
            ('webpack', webpack_config),
        ]

    if workers:
        if settings.CELERY_ALWAYS_EAGER:
            raise click.ClickException(
                'Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.'
            )

        daemons += [
            ('worker', ['sentry', 'run', 'worker', '-c', '1', '--autoreload']),
            ('cron', ['sentry', 'run', 'cron', '--autoreload']),
        ]

        from sentry import eventstream
        if eventstream.requires_post_process_forwarder():
            daemons += [
                ('relay', ['sentry', 'run', 'post-process-forwarder', '--loglevel=debug', '--commit-batch-size=1']),
            ]

    if needs_https and has_https:
        https_port = six.text_type(parsed_url.port)
        https_host = parsed_url.hostname

        # Determine a random port for the backend http server
        import socket
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        s.bind((host, 0))
        port = s.getsockname()[1]
        s.close()
        bind = '%s:%d' % (host, port)

        daemons += [
            ('https', ['https', '-host', https_host, '-listen', host + ':' + https_port, bind]),
        ]

    # A better log-format for local dev when running through honcho,
    # but if there aren't any other daemons, we don't want to override.
    if daemons:
        uwsgi_overrides['log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
    else:
        uwsgi_overrides['log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'

    server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides)

    # If we don't need any other daemons, just launch a normal uwsgi webserver
    # and avoid dealing with subprocesses
    if not daemons:
        return server.run()

    import sys
    from subprocess import list2cmdline
    from honcho.manager import Manager
    from honcho.printer import Printer

    os.environ['PYTHONUNBUFFERED'] = 'true'

    # Make sure that the environment is prepared before honcho takes over
    # This sets all the appropriate uwsgi env vars, etc
    server.prepare_environment()
    daemons += [
        ('server', ['sentry', 'run', 'web']),
    ]

    if styleguide:
        daemons += [('storybook', ['yarn', 'storybook'])]

    cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))

    manager = Manager(Printer(prefix=prefix))
    for name, cmd in daemons:
        manager.add_process(
            name,
            list2cmdline(cmd),
            quiet=False,
            cwd=cwd,
        )

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 21
0
class TestManager(object):
    @pytest.fixture(autouse=True)
    def printer(self):  # noqa
        self.p = FakePrinter()
        self.m = Manager(printer=self.p)
        self.m._env = FakeEnv()

    def run_history(self, name, wait=True):
        self.h = Harness(HISTORIES[name], self.m)
        self.h.run(wait=wait)

    def test_init_sets_default_printer_width(self):
        assert self.p.width == len(SYSTEM_PRINTER_NAME)

    def test_add_process_updates_printer_width(self):
        self.m.add_process('interesting', 'ruby server.rb')
        assert self.p.width == len('interesting')

    def test_add_process_sets_name(self):
        proc = self.m.add_process('foo', 'ruby server.rb')
        assert proc.name == 'foo'

    def test_add_process_sets_cmd(self):
        proc = self.m.add_process('foo', 'ruby server.rb')
        assert proc.cmd == 'ruby server.rb'

    def test_add_process_sets_colour(self):
        proc = self.m.add_process('foo', 'ruby server.rb')
        assert proc.colour is not None

    def test_add_process_sets_unique_colours(self):
        p1 = self.m.add_process('foo', 'ruby server.rb')
        p2 = self.m.add_process('bar', 'python server.py')
        assert p1.colour != p2.colour

    def test_add_process_sets_quiet(self):
        proc = self.m.add_process('foo', 'ruby server.rb', quiet=True)
        assert proc.quiet

    def test_add_process_name_must_be_unique(self):
        self.m.add_process('foo', 'ruby server.rb')
        with pytest.raises(AssertionError):
            self.m.add_process('foo', 'another command')

    def test_add_process_sets_cwd(self):
        proc = self.m.add_process('foo', 'ruby server.rb', cwd='foo-dir')
        assert proc.cwd == 'foo-dir'

    def test_loop_with_empty_manager_returns_immediately(self):
        self.m.loop()

    def test_loop_calls_process_run(self):
        self.run_history('one')
        evts = self.h.find_events(type='run')
        assert len(evts) == 1
        assert evts[0]['name'] == 'foo'
        assert evts[0]['events_passed']

    def test_printer_receives_messages_in_correct_order(self):
        self.run_history('one')
        self.p.fetch_lines()
        assert self.p.lines_local[0].data == 'foo started (pid=123)\n'
        assert self.p.lines_local[1].data == b'hello, world!\n'
        assert self.p.lines_local[2].data == 'foo stopped (rc=0)\n'

    def test_printer_receives_lines_multi_process(self):
        self.run_history('two')
        l1 = self.p.find_line(b'process one\n')
        l2 = self.p.find_line(b'process two\n')
        assert l1.name == 'foo'
        assert l2.name == 'bar'

    def test_returncode_set_by_first_exiting_process(self):
        self.run_history('returncode')
        assert self.h.manager_returncode == 456

    def test_printer_receives_lines_after_stop(self):
        self.run_history('output_after_stop')
        assert self.p.got_line(b'fishmongers\n')
        assert self.p.got_line(b'butchers\n')
Esempio n. 22
0
class TestManager(TestCase):

    def setUp(self):  # noqa
        self.p = FakePrinter()
        self.m = Manager(printer=self.p)
        self.m._env = FakeEnv()

    def run_history(self, name, wait=True):
        self.h = Harness(HISTORIES[name], self.m)
        self.h.run(wait=wait)

    def test_init_sets_default_printer_width(self):
        self.assertEqual(len(SYSTEM_PRINTER_NAME), self.p.width)

    def test_add_process_updates_printer_width(self):
        self.m.add_process('interesting', 'ruby server.rb')
        self.assertEqual(len('interesting'), self.p.width)

    def test_add_process_sets_name(self):
        proc = self.m.add_process('foo', 'ruby server.rb')
        self.assertEqual('foo', proc.name)

    def test_add_process_sets_cmd(self):
        proc = self.m.add_process('foo', 'ruby server.rb')
        self.assertEqual('ruby server.rb', proc.cmd)

    def test_add_process_sets_colour(self):
        proc = self.m.add_process('foo', 'ruby server.rb')
        self.assertTrue(proc.colour is not None)

    def test_add_process_sets_unique_colours(self):
        p1 = self.m.add_process('foo', 'ruby server.rb')
        p2 = self.m.add_process('bar', 'python server.py')
        self.assertNotEqual(p1.colour, p2.colour)

    def test_add_process_sets_quiet(self):
        proc = self.m.add_process('foo', 'ruby server.rb', quiet=True)
        self.assertTrue(proc.quiet)

    def test_add_process_name_must_be_unique(self):
        self.m.add_process('foo', 'ruby server.rb')
        self.assertRaises(AssertionError, self.m.add_process, 'foo', 'another command')

    def test_loop_with_empty_manager_returns_immediately(self):
        self.m.loop()

    def test_loop_calls_process_run(self):
        self.run_history('one')
        evts = self.h.find_events(type='run')
        self.assertEqual(1, len(evts))
        self.assertEqual('foo', evts[0]['name'])
        self.assertTrue(evts[0]['events_passed'])

    def test_printer_receives_messages_in_correct_order(self):
        self.run_history('one')
        self.p.fetch_lines()
        self.assertEqual('foo started (pid=123)\n', self.p.lines_local[0].data)
        self.assertEqual(b'hello, world!\n', self.p.lines_local[1].data)
        self.assertEqual('foo stopped (rc=0)\n', self.p.lines_local[2].data)

    def test_printer_receives_lines_multi_process(self):
        self.run_history('two')
        l1 = self.p.find_line(b'process one\n')
        l2 = self.p.find_line(b'process two\n')
        self.assertEqual('foo', l1.name)
        self.assertEqual('bar', l2.name)

    def test_returncode_set_by_first_exiting_process(self):
        self.run_history('returncode')
        self.assertEqual(456, self.h.manager_returncode)

    def test_printer_receives_lines_after_stop(self):
        self.run_history('output_after_stop')
        self.assertTrue(self.p.got_line(b'fishmongers\n'))
        self.assertTrue(self.p.got_line(b'butchers\n'))
Esempio n. 23
0
def devserver(reload, watchers, workers, browser_reload, environment, bind):
    "Starts a lightweight web server for development."
    if ':' in bind:
        host, port = bind.split(':', 1)
        port = int(port)
    else:
        host = bind
        port = None

    import os

    os.environ['SENTRY_ENVIRONMENT'] = environment

    from django.conf import settings
    from sentry import options
    from sentry.services.http import SentryHTTPServer

    url_prefix = options.get('system.url-prefix', '')
    needs_https = url_prefix.startswith('https://')
    has_https = False

    if needs_https:
        from subprocess import check_output
        try:
            check_output(['which', 'https'])
            has_https = True
        except Exception:
            has_https = False
            from sentry.runner.initializer import show_big_error
            show_big_error([
                'missing `https` on your `$PATH`, but https is needed',
                '`$ brew install mattrobenolt/stuff/https`',
            ])

    uwsgi_overrides = {
        # Make sure we don't try and use uwsgi protocol
        'protocol': 'http',
        # Make sure we reload really quickly for local dev in case it
        # doesn't want to shut down nicely on it's own, NO MERCY
        'worker-reload-mercy': 2,
        # We need stdin to support pdb in devserver
        'honour-stdin': True,
    }

    if reload:
        uwsgi_overrides['py-autoreload'] = 1

    daemons = []

    if watchers:
        daemons += settings.SENTRY_WATCHERS

    if workers:
        if settings.CELERY_ALWAYS_EAGER:
            raise click.ClickException(
                'Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.'
            )

        daemons += [
            ('worker', ['sentry', 'run', 'worker', '-c', '1', '--autoreload']),
            ('cron', ['sentry', 'run', 'cron', '--autoreload']),
        ]

    if needs_https and has_https:
        from six.moves.urllib.parse import urlparse
        parsed_url = urlparse(url_prefix)
        https_port = six.text_type(parsed_url.port or 443)
        https_host = parsed_url.hostname

        # Determine a random port for the backend http server
        import socket
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        s.bind((host, 0))
        port = s.getsockname()[1]
        s.close()
        bind = '%s:%d' % (host, port)

        daemons += [
            ('https', [
                'https', '-host', https_host, '-listen',
                host + ':' + https_port, bind
            ]),
        ]

    # A better log-format for local dev when running through honcho,
    # but if there aren't any other daemons, we don't want to override.
    if daemons:
        uwsgi_overrides[
            'log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
    else:
        uwsgi_overrides[
            'log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'

    server = SentryHTTPServer(host=host,
                              port=port,
                              workers=1,
                              extra_options=uwsgi_overrides)

    # If we don't need any other daemons, just launch a normal uwsgi webserver
    # and avoid dealing with subprocesses
    if not daemons:
        return server.run()

    import sys
    from subprocess import list2cmdline
    from honcho.manager import Manager

    os.environ['PYTHONUNBUFFERED'] = 'true'
    if browser_reload:
        os.environ['WEBPACK_LIVERELOAD'] = '1'

    # Make sure that the environment is prepared before honcho takes over
    # This sets all the appropriate uwsgi env vars, etc
    server.prepare_environment()
    daemons += [
        ('server', ['sentry', 'run', 'web']),
    ]

    cwd = os.path.realpath(
        os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))

    manager = Manager()
    for name, cmd in daemons:
        manager.add_process(
            name,
            list2cmdline(cmd),
            quiet=False,
            cwd=cwd,
        )

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 24
0
def devserver(*, bootstrap: bool, workers: bool) -> None:
    "Starts all Snuba processes for local development."
    import os
    import sys
    from subprocess import call, list2cmdline

    from honcho.manager import Manager

    os.environ["PYTHONUNBUFFERED"] = "1"

    if bootstrap:
        cmd = ["snuba", "bootstrap", "--force", "--no-migrate"]
        if not workers:
            cmd.append("--no-kafka")
        returncode = call(cmd)
        if returncode > 0:
            sys.exit(returncode)

        # Run migrations
        returncode = call(["snuba", "migrations", "migrate", "--force"])
        if returncode > 0:
            sys.exit(returncode)

    daemons = [("api", ["snuba", "api"])]

    if not workers:
        os.execvp(daemons[0][1][0], daemons[0][1])

    daemons += [
        (
            "transaction-consumer",
            [
                "snuba",
                "consumer",
                "--auto-offset-reset=latest",
                "--no-strict-offset-reset",
                "--log-level=debug",
                "--storage=transactions",
                "--consumer-group=transactions_group",
                "--commit-log-topic=snuba-commit-log",
            ],
        ),
        (
            "sessions-consumer",
            [
                "snuba",
                "consumer",
                "--auto-offset-reset=latest",
                "--no-strict-offset-reset",
                "--log-level=debug",
                "--storage=sessions_raw",
                "--consumer-group=sessions_group",
            ],
        ),
        (
            "outcomes-consumer",
            [
                "snuba",
                "consumer",
                "--auto-offset-reset=latest",
                "--no-strict-offset-reset",
                "--log-level=debug",
                "--storage=outcomes_raw",
                "--consumer-group=outcomes_group",
            ],
        ),
        (
            "consumer",
            [
                "snuba",
                "consumer",
                "--auto-offset-reset=latest",
                "--no-strict-offset-reset",
                "--log-level=debug",
                "--storage=errors",
            ],
        ),
        (
            "replacer",
            [
                "snuba",
                "replacer",
                "--auto-offset-reset=latest",
                "--no-strict-offset-reset",
                "--log-level=debug",
                "--storage=errors",
            ],
        ),
        (
            "cdc-consumer",
            [
                "snuba",
                "multistorage-consumer",
                "--auto-offset-reset=latest",
                "--no-strict-offset-reset",
                "--log-level=debug",
                "--storage=groupedmessages",
                "--storage=groupassignees",
                "--consumer-group=cdc_group",
            ],
        ),
    ]

    if settings.SEPARATE_SCHEDULER_EXECUTOR_SUBSCRIPTIONS_DEV:
        daemons += [
            (
                "subscriptions-scheduler-events",
                [
                    "snuba",
                    "subscriptions-scheduler",
                    "--entity=events",
                    "--consumer-group=snuba-events-subscriptions-scheduler",
                    "--followed-consumer-group=snuba-consumers",
                    "--auto-offset-reset=latest",
                    "--log-level=debug",
                    "--delay-seconds=1",
                    "--schedule-ttl=10",
                ],
            ),
            (
                "subscriptions-executor-events",
                [
                    "snuba",
                    "subscriptions-executor",
                    "--dataset=events",
                    "--entity=events",
                    "--consumer-group=snuba-events-subscription-executor",
                    "--auto-offset-reset=latest",
                ],
            ),
            (
                "subscriptions-scheduler-transactions",
                [
                    "snuba",
                    "subscriptions-scheduler",
                    "--entity=transactions",
                    "--consumer-group=snuba-transactions-subscriptions-scheduler",
                    "--followed-consumer-group=transactions_group",
                    "--auto-offset-reset=latest",
                    "--log-level=debug",
                    "--delay-seconds=1",
                    "--schedule-ttl=10",
                ],
            ),
            (
                "subscriptions-executor-transactions",
                [
                    "snuba",
                    "subscriptions-executor",
                    "--dataset=transactions",
                    "--entity=transactions",
                    "--consumer-group=snuba-transactions-subscription-executor",
                    "--auto-offset-reset=latest",
                ],
            ),
        ]

    else:
        daemons += [
            (
                "subscriptions-scheduler-executor-events",
                [
                    "snuba",
                    "subscriptions-scheduler-executor",
                    "--dataset=events",
                    "--entity=events",
                    "--consumer-group=snuba-events-subscriptions-scheduler-executor",
                    "--followed-consumer-group=snuba-consumers",
                    "--auto-offset-reset=latest",
                    "--no-strict-offset-reset",
                    "--log-level=debug",
                    "--delay-seconds=1",
                    "--schedule-ttl=10",
                    "--stale-threshold-seconds=900",
                ],
            ),
            (
                "subscriptions-scheduler-executor-transactions",
                [
                    "snuba",
                    "subscriptions-scheduler-executor",
                    "--dataset=transactions",
                    "--entity=transactions",
                    "--consumer-group=snuba-transactions-subscriptions-scheduler-executor",
                    "--followed-consumer-group=transactions_group",
                    "--auto-offset-reset=latest",
                    "--no-strict-offset-reset",
                    "--log-level=debug",
                    "--delay-seconds=1",
                    "--schedule-ttl=10",
                    "--stale-threshold-seconds=900",
                ],
            ),
        ]

    if settings.ENABLE_SENTRY_METRICS_DEV:
        daemons += [
            (
                "metrics-consumer",
                [
                    "snuba",
                    "consumer",
                    "--storage=metrics_raw",
                    "--auto-offset-reset=latest",
                    "--no-strict-offset-reset",
                    "--log-level=debug",
                    "--consumer-group=snuba-metrics-consumers",
                ],
            ),
        ]
        if settings.ENABLE_METRICS_SUBSCRIPTIONS:
            if settings.SEPARATE_SCHEDULER_EXECUTOR_SUBSCRIPTIONS_DEV:
                daemons += [
                    (
                        "subscriptions-scheduler-metrics-counters",
                        [
                            "snuba",
                            "subscriptions-scheduler",
                            "--entity=metrics_counters",
                            "--consumer-group=snuba-metrics-subscriptions-scheduler",
                            "--followed-consumer-group=snuba-metrics-consumers",
                            "--auto-offset-reset=latest",
                            "--log-level=debug",
                            "--delay-seconds=1",
                            "--schedule-ttl=10",
                        ],
                    ),
                    (
                        "subscriptions-scheduler-metrics-sets",
                        [
                            "snuba",
                            "subscriptions-scheduler",
                            "--entity=metrics_sets",
                            "--consumer-group=snuba-metrics-subscriptions-scheduler",
                            "--followed-consumer-group=snuba-metrics-consumers",
                            "--auto-offset-reset=latest",
                            "--log-level=debug",
                            "--delay-seconds=1",
                            "--schedule-ttl=10",
                        ],
                    ),
                    (
                        "subscriptions-executor-metrics",
                        [
                            "snuba",
                            "subscriptions-executor",
                            "--dataset=metrics",
                            "--entity=metrics_counters",
                            "--entity=metrics_sets",
                            "--consumer-group=snuba-metrics-subscription-executor",
                            "--auto-offset-reset=latest",
                        ],
                    ),
                ]
            else:
                daemons += [
                    (
                        "subscriptions-scheduler-executor-metrics",
                        [
                            "snuba",
                            "subscriptions-scheduler-executor",
                            "--dataset=metrics",
                            "--entity=metrics_sets",
                            "--entity=metrics_counters",
                            "--consumer-group=snuba-metrics-subscriptions-scheduler-executor",
                            "--followed-consumer-group=snuba-metrics-consumers",
                            "--auto-offset-reset=latest",
                            "--no-strict-offset-reset",
                            "--log-level=debug",
                            "--delay-seconds=1",
                            "--schedule-ttl=10",
                        ],
                    ),
                ]

    if settings.ENABLE_PROFILES_CONSUMER:
        daemons += [
            (
                "profiles",
                [
                    "snuba",
                    "consumer",
                    "--auto-offset-reset=latest",
                    "--no-strict-offset-reset",
                    "--log-level=debug",
                    "--storage=profiles",
                ],
            ),
        ]
    if settings.ENABLE_REPLAYS_CONSUMER:
        daemons += [
            (
                "replays-consumer",
                [
                    "snuba",
                    "consumer",
                    "--auto-offset-reset=latest",
                    "--log-level=debug",
                    "--storage=replays",
                ],
            ),
        ]

    manager = Manager()
    for name, cmd in daemons:
        manager.add_process(
            name,
            list2cmdline(cmd),
            quiet=False,
        )

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 25
0
 def printer(self):  # noqa
     self.p = FakePrinter()
     self.m = Manager(printer=self.p)
     self.m._env = FakeEnv()
Esempio n. 26
0
def devserver(https, web, ws, worker, assets, beat):
    """
    Run a development server.

    This command will start a development instance of h, consisting of a web
    application, Celery worker, and websocket server. It will also start a
    process which will watch and build the frontend assets.

    By default, the webserver will be accessible at:

        http://localhost:5000

    You can also pass the `--https` flag, which will look for a TLS certificate
    and key in PEM format in the current directory, in files called:

        .tlscert.pem
        .tlskey.pem

    If you use this flag, the webserver will be accessible at:

        https://localhost:5000

    If you wish this to be the default behaviour, you can set the
    USE_HTTPS environment variable.
    """
    try:
        from honcho.manager import Manager
    except ImportError:
        raise click.ClickException(
            'cannot import honcho: did you run `pip install -r requirements-dev.in` yet?'
        )

    os.environ['PYTHONUNBUFFERED'] = 'true'
    if https:
        gunicorn_args = '--certfile=.tlscert.pem --keyfile=.tlskey.pem'
        os.environ['APP_URL'] = 'https://localhost:5000'
        os.environ['WEBSOCKET_URL'] = 'wss://localhost:5001/ws'
        os.environ['ALLOWED_ORIGINS'] = ' '.join(
            ['https://localhost:5000',
             os.environ.get('ALLOWED_ORIGINS', '')])
    else:
        gunicorn_args = ''
        os.environ['APP_URL'] = 'http://localhost:5000'
        os.environ['WEBSOCKET_URL'] = 'ws://localhost:5001/ws'

    m = Manager()
    if web:
        m.add_process(
            'web', 'gunicorn --reload --paste conf/development-app.ini %s' %
            gunicorn_args)

    if ws:
        m.add_process(
            'ws',
            'gunicorn --reload --paste conf/development-websocket.ini %s' %
            gunicorn_args)

    if worker:
        m.add_process('worker', 'hypothesis --dev celery worker --autoreload')

    if beat:
        m.add_process('beat', 'hypothesis --dev celery beat')

    if assets:
        m.add_process('assets', 'gulp watch')

    m.loop()

    sys.exit(m.returncode)
Esempio n. 27
0
def devserver(reload, watchers, workers, bind):
    "Starts a lightweight web server for development."
    if ':' in bind:
        host, port = bind.split(':', 1)
        port = int(port)
    else:
        host = bind
        port = None

    import os
    from django.conf import settings
    from sentry.services.http import SentryHTTPServer

    uwsgi_overrides = {
        # Make sure we don't try and use uwsgi protocol
        'protocol': 'http',
        # A better log-format for local dev
        'log-format': '"%(method) %(uri) %(proto)" %(status) %(size) "%(referer)" "%(uagent)"'
    }

    if reload:
        uwsgi_overrides['py-autoreload'] = 1

    server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides)

    daemons = []

    if watchers:
        daemons += settings.SENTRY_WATCHERS

    if workers:
        if settings.CELERY_ALWAYS_EAGER:
            raise click.ClickException('Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.')

        daemons += [
            ('worker', ['sentry', 'celery', 'worker', '-c', '1', '-l', 'INFO']),
            ('beat', ['sentry', 'celery', 'beat', '-l', 'INFO']),
        ]

    # If we don't need any other daemons, just launch a normal uwsgi webserver
    # and avoid dealing with subprocesses
    if not daemons:
        return server.run()

    import sys
    from subprocess import list2cmdline
    from honcho.manager import Manager

    os.environ['PYTHONUNBUFFERED'] = 'true'

    # Make sure that the environment is prepared before honcho takes over
    # This sets all the appropriate uwsgi env vars, etc
    server.prepare_environment()
    daemons += [
        ('server', ['sentry', 'start']),
    ]

    # Change directory globally to affect all subprocesses. This is less than ideal,
    # but Honcho doesn't provide the ability to pass in a cwd to subprocesses yet.
    # See: https://github.com/nickstenning/honcho/pull/170
    cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))
    os.chdir(cwd)

    manager = Manager()
    for name, cmd in daemons:
        manager.add_process(
            name, list2cmdline(cmd),
            quiet=False, env=os.environ.copy()
        )

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 28
0
def devserver(*, bootstrap: bool, workers: bool) -> None:
    "Starts all Snuba processes for local development."
    import os
    import sys
    from subprocess import call, list2cmdline

    from honcho.manager import Manager

    os.environ["PYTHONUNBUFFERED"] = "1"

    if bootstrap:
        cmd = ["snuba", "bootstrap", "--force", "--no-migrate"]
        if not workers:
            cmd.append("--no-kafka")
        returncode = call(cmd)
        if returncode > 0:
            sys.exit(returncode)

        # Run migrations
        returncode = call(["snuba", "migrations", "migrate", "--force"])
        if returncode > 0:
            sys.exit(returncode)

    daemons = [("api", ["snuba", "api"])]

    if not workers:
        os.execvp(daemons[0][1][0], daemons[0][1])

    daemons += [
        (
            "transaction-consumer",
            [
                "snuba",
                "consumer",
                "--auto-offset-reset=latest",
                "--log-level=debug",
                "--storage=transactions",
                "--consumer-group=transactions_group",
                "--commit-log-topic=snuba-commit-log",
            ],
        ),
        (
            "sessions-consumer",
            [
                "snuba",
                "consumer",
                "--auto-offset-reset=latest",
                "--log-level=debug",
                "--storage=sessions_raw",
                "--consumer-group=sessions_group",
            ],
        ),
        (
            "outcomes-consumer",
            [
                "snuba",
                "consumer",
                "--auto-offset-reset=latest",
                "--log-level=debug",
                "--storage=outcomes_raw",
                "--consumer-group=outcomes_group",
            ],
        ),
        (
            "consumer",
            [
                "snuba",
                "consumer",
                "--auto-offset-reset=latest",
                "--log-level=debug",
                "--storage=errors",
            ],
        ),
        (
            "replacer",
            [
                "snuba",
                "replacer",
                "--auto-offset-reset=latest",
                "--log-level=debug",
                "--storage=errors",
            ],
        ),
        (
            "subscriptions-consumer-events",
            [
                "snuba",
                "subscriptions",
                "--auto-offset-reset=latest",
                "--log-level=debug",
                "--max-batch-size=1",
                "--consumer-group=snuba-events-subscriptions-consumers",
                "--dataset=events",
                "--commit-log-topic=snuba-commit-log",
                "--commit-log-group=snuba-consumers",
                "--delay-seconds=1",
                "--schedule-ttl=10",
                "--max-query-workers=1",
            ],
        ),
        (
            "subscriptions-consumer-transactions",
            [
                "snuba",
                "subscriptions",
                "--auto-offset-reset=latest",
                "--log-level=debug",
                "--max-batch-size=1",
                "--consumer-group=snuba-transactions-subscriptions-consumers",
                "--dataset=transactions",
                "--commit-log-topic=snuba-commit-log",
                "--commit-log-group=transactions_group",
                "--delay-seconds=1",
                "--schedule-ttl=10",
                "--max-query-workers=1",
            ],
        ),
        (
            "cdc-consumer",
            [
                "snuba",
                "multistorage-consumer",
                "--auto-offset-reset=latest",
                "--log-level=debug",
                "--storage=groupedmessages",
                "--storage=groupassignees",
                "--consumer-group=cdc_group",
            ],
        ),
    ]

    manager = Manager()
    for name, cmd in daemons:
        manager.add_process(
            name, list2cmdline(cmd), quiet=False,
        )

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 29
0
def devserver(
    environment,
    workers,
    host,
    port,
    ngrok,
    ngrok_domain,
    vcs_server,
    vcs_server_port,
    pubsub,
    pubsub_port,
    watchdog,
):
    os.environ.setdefault("FLASK_DEBUG", "1")
    os.environ["NODE_ENV"] = environment

    if vcs_server:
        os.environ["VCS_SERVER_ENDPOINT"] = "http://localhost:{}".format(
            vcs_server_port)

    if pubsub:
        os.environ["PUBSUB_ENDPOINT"] = "http://localhost:{}".format(
            pubsub_port)

    if ngrok:
        root_url = "https://{}.ngrok.io".format(ngrok_domain)
        os.environ["SSL"] = "1"
        os.environ["SERVER_NAME"] = "{}.ngrok.io".format(ngrok_domain)
    else:
        root_url = "http://{}:{}".format(host, port)

    click.echo("Launching Zeus on {}".format(root_url))

    # TODO(dcramer): pass required attributes to 'run' directly instead
    # of relying on FLASK_DEBUG
    daemons = [
        ("web",
         ["zeus", "run", "--host={}".format(host), "--port={}".format(port)]),
        (
            "webpack",
            [
                "node_modules/.bin/webpack",
                "--watch",
                "--config=config/webpack.config.js",
            ],
        ),
    ]

    if vcs_server:
        daemons.append((
            "vcs-server",
            [
                "zeus",
                "vcs-server",
                "--host={}".format(host),
                "--port={}".format(vcs_server_port),
            ],
        ))

    if pubsub:
        daemons.append((
            "pubsub",
            [
                "zeus",
                "pubsub",
                "--host={}".format(host),
                "--port={}".format(pubsub_port),
            ],
        ))

    if workers:
        daemons.append(
            ("worker", ["zeus", "worker", "--cron", "--log-level=INFO"]))

    if ngrok:
        daemons.append((
            "ngrok",
            ["ngrok", "http", "-subdomain={}".format(ngrok_domain),
             str(port)],
        ))

    cwd = os.path.realpath(
        os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))

    manager = Manager()
    for name, cmd in daemons:
        manager.add_process(name, list2cmdline(cmd), quiet=False, cwd=cwd)

    if watchdog:
        event_handler = LoggingEventHandler()
        observer = Observer()
        observer.schedule(
            event_handler,
            os.path.join(os.path.dirname(__file__), os.pardir),
            recursive=True,
        )
        observer.start()

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 30
0
def devserver(reload, watchers, workers, browser_reload, environment, bind):
    "Starts a lightweight web server for development."
    if ':' in bind:
        host, port = bind.split(':', 1)
        port = int(port)
    else:
        host = bind
        port = None

    import os

    os.environ['SENTRY_ENVIRONMENT'] = environment

    from django.conf import settings
    from sentry import options
    from sentry.services.http import SentryHTTPServer

    url_prefix = options.get('system.url-prefix', '')
    needs_https = url_prefix.startswith('https://')
    has_https = False

    if needs_https:
        from subprocess import check_output
        try:
            check_output(['which', 'https'])
            has_https = True
        except Exception:
            has_https = False
            from sentry.runner.initializer import show_big_error
            show_big_error([
                'missing `https` on your `$PATH`, but https is needed',
                '`$ brew install mattrobenolt/stuff/https`',
            ])

    uwsgi_overrides = {
        # Make sure we don't try and use uwsgi protocol
        'protocol': 'http',
        # Make sure we reload really quickly for local dev in case it
        # doesn't want to shut down nicely on it's own, NO MERCY
        'worker-reload-mercy': 2,
        # We need stdin to support pdb in devserver
        'honour-stdin': True,
    }

    if reload:
        uwsgi_overrides['py-autoreload'] = 1

    daemons = []

    if watchers:
        daemons += settings.SENTRY_WATCHERS

    if workers:
        if settings.CELERY_ALWAYS_EAGER:
            raise click.ClickException('Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.')

        daemons += [
            ('worker', ['sentry', 'run', 'worker', '-c', '1', '--autoreload']),
            ('cron', ['sentry', 'run', 'cron', '--autoreload']),
        ]

    if needs_https and has_https:
        from six.moves.urllib.parse import urlparse
        parsed_url = urlparse(url_prefix)
        https_port = six.text_type(parsed_url.port or 443)
        https_host = parsed_url.hostname

        # Determine a random port for the backend http server
        import socket
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        s.bind((host, 0))
        port = s.getsockname()[1]
        s.close()
        bind = '%s:%d' % (host, port)

        daemons += [
            ('https', ['https', '-host', https_host, '-listen', host + ':' + https_port, bind]),
        ]

    # A better log-format for local dev when running through honcho,
    # but if there aren't any other daemons, we don't want to override.
    if daemons:
        uwsgi_overrides['log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
    else:
        uwsgi_overrides['log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'

    server = SentryHTTPServer(host=host, port=port, workers=1, extra_options=uwsgi_overrides)

    # If we don't need any other daemons, just launch a normal uwsgi webserver
    # and avoid dealing with subprocesses
    if not daemons:
        return server.run()

    import sys
    from subprocess import list2cmdline
    from honcho.manager import Manager

    os.environ['PYTHONUNBUFFERED'] = 'true'
    if browser_reload:
        os.environ['WEBPACK_LIVERELOAD'] = '1'

    # Make sure that the environment is prepared before honcho takes over
    # This sets all the appropriate uwsgi env vars, etc
    server.prepare_environment()
    daemons += [
        ('server', ['sentry', 'run', 'web']),
    ]

    cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))

    manager = Manager()
    for name, cmd in daemons:
        manager.add_process(
            name, list2cmdline(cmd),
            quiet=False, cwd=cwd,
        )

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 31
0
def devserver(*, bootstrap: bool, workers: bool) -> None:
    "Starts all Snuba processes for local development."
    import os
    import sys
    from subprocess import list2cmdline, call
    from honcho.manager import Manager

    os.environ["PYTHONUNBUFFERED"] = "1"

    if bootstrap:
        cmd = ["snuba", "bootstrap", "--force"]
        if not workers:
            cmd.append("--no-kafka")
        returncode = call(cmd)
        if returncode > 0:
            sys.exit(returncode)

    daemons = [
        (
            "api",
            [
                "uwsgi",
                "--master",
                "--manage-script-name",
                "--wsgi-file",
                "snuba/web/wsgi.py",
                "--http",
                "0.0.0.0:1218",
                "--http-keepalive",
                "--need-app",
                "--die-on-term",
            ],
        ),
    ]

    if not workers:
        os.execvp(daemons[0][1][0], daemons[0][1])

    daemons += [
        (
            "transaction-consumer",
            [
                "snuba",
                "consumer",
                "--auto-offset-reset=latest",
                "--log-level=debug",
                "--dataset=transactions",
                "--consumer-group=transactions_group",
            ],
        ),
        (
            "consumer",
            [
                "snuba", "consumer", "--auto-offset-reset=latest",
                "--log-level=debug"
            ],
        ),
        (
            "replacer",
            [
                "snuba", "replacer", "--auto-offset-reset=latest",
                "--log-level=debug"
            ],
        ),
    ]

    manager = Manager()
    for name, cmd in daemons:
        manager.add_process(
            name,
            list2cmdline(cmd),
            quiet=False,
        )

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 32
0
 def setUp(self):  # noqa
     self.p = FakePrinter()
     self.m = Manager(printer=self.p)
     self.m._env = FakeEnv()
Esempio n. 33
0
def devserver(https, web, ws, worker, assets, beat):
    """
    Run a development server.

    This command will start a development instance of h, consisting of a web
    application, Celery worker, and websocket server. It will also start a
    process which will watch and build the frontend assets.

    By default, the webserver will be accessible at:

        http://localhost:5000

    You can also pass the `--https` flag, which will look for a TLS certificate
    and key in PEM format in the current directory, in files called:

        .tlscert.pem
        .tlskey.pem

    If you use this flag, the webserver will be accessible at:

        https://localhost:5000

    If you wish this to be the default behaviour, you can set the
    USE_HTTPS environment variable.
    """
    try:
        from honcho.manager import Manager
    except ImportError:
        raise click.ClickException(
            "cannot import honcho: did you run `pip install -r requirements-dev.in` yet?"
        )

    os.environ["PYTHONUNBUFFERED"] = "true"
    if https:
        gunicorn_args = "--certfile=.tlscert.pem --keyfile=.tlskey.pem"
        os.environ["APP_URL"] = "https://localhost:5000"
        os.environ["WEBSOCKET_URL"] = "wss://localhost:5001/ws"
    else:
        gunicorn_args = ""
        os.environ["APP_URL"] = "http://localhost:5000"
        os.environ["WEBSOCKET_URL"] = "ws://localhost:5001/ws"

    m = Manager()
    if web:
        m.add_process(
            "web",
            "newrelic-admin run-program gunicorn --name web --reload --paste conf/development-app.ini %s"
            % gunicorn_args,
        )

    if ws:
        m.add_process(
            "ws",
            "newrelic-admin run-program gunicorn --name websocket --reload --paste conf/development-websocket.ini %s"
            % gunicorn_args,
        )

    if worker:
        m.add_process("worker", "hypothesis --dev celery worker -l INFO")

    if beat:
        m.add_process("beat", "hypothesis --dev celery beat")

    if assets:
        m.add_process("assets", "gulp watch")

    m.loop()

    sys.exit(m.returncode)
Esempio n. 34
0
def devserver(
    reload,
    watchers,
    workers,
    experimental_spa,
    styleguide,
    prefix,
    pretty,
    environment,
    debug_server,
    bind,
):
    "Starts a lightweight web server for development."

    if bind is None:
        bind = "127.0.0.1:8000"

    if ":" in bind:
        host, port = bind.split(":", 1)
        port = int(port)
    else:
        host = bind
        port = None

    import os

    os.environ["SENTRY_ENVIRONMENT"] = environment
    # NODE_ENV *must* use production for any prod-like environment as third party libraries look
    # for this magic constant
    os.environ["NODE_ENV"] = "production" if environment.startswith(
        "prod") else environment

    from django.conf import settings
    from sentry import options
    from sentry.services.http import SentryHTTPServer

    url_prefix = options.get("system.url-prefix", "")
    parsed_url = urlparse(url_prefix)
    # Make sure we're trying to use a port that we can actually bind to
    needs_https = parsed_url.scheme == "https" and (parsed_url.port
                                                    or 443) > 1024
    has_https = False

    if needs_https:
        from subprocess import check_output

        try:
            check_output(["which", "https"])
            has_https = True
        except Exception:
            has_https = False
            from sentry.runner.initializer import show_big_error

            show_big_error([
                "missing `https` on your `$PATH`, but https is needed",
                "`$ brew install mattrobenolt/stuff/https`",
            ])

    uwsgi_overrides = {
        "http-keepalive": True,
        # Make sure we reload really quickly for local dev in case it
        # doesn't want to shut down nicely on it's own, NO MERCY
        "worker-reload-mercy": 2,
        # We need stdin to support pdb in devserver
        "honour-stdin": True,
        # accept ridiculously large files
        "limit-post": 1 << 30,
        # do something with chunked
        "http-chunked-input": True,
        "thunder-lock": False,
        "timeout": 600,
        "harakiri": 600,
    }

    if reload:
        uwsgi_overrides["py-autoreload"] = 1

    daemons = []

    if experimental_spa:
        os.environ["SENTRY_UI_DEV_ONLY"] = "1"
        if not watchers:
            click.secho(
                "Using experimental SPA mode without watchers enabled has no effect",
                err=True,
                fg="yellow",
            )

    # We proxy all requests through webpacks devserver on the configured port.
    # The backend is served on port+1 and is proxied via the webpack
    # configuration.
    if watchers:
        daemons += settings.SENTRY_WATCHERS

        proxy_port = port
        port = port + 1

        uwsgi_overrides["protocol"] = "http"

        os.environ["FORCE_WEBPACK_DEV_SERVER"] = "1"
        os.environ["SENTRY_WEBPACK_PROXY_PORT"] = "%s" % proxy_port
        os.environ["SENTRY_BACKEND_PORT"] = "%s" % port

        # webpack and/or typescript is causing memory issues
        os.environ["NODE_OPTIONS"] = ((os.environ.get("NODE_OPTIONS", "") +
                                       " --max-old-space-size=4096")).lstrip()

        # Replace the webpack watcher with the drop-in webpack-dev-server
        webpack_config = next(w for w in daemons if w[0] == "webpack")[1]
        webpack_config[0] = os.path.join(
            *os.path.split(webpack_config[0])[0:-1] + ("webpack-dev-server", ))

        daemons = [w for w in daemons if w[0] != "webpack"
                   ] + [("webpack", webpack_config)]
    else:
        # If we are the bare http server, use the http option with uwsgi protocol
        # See https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
        uwsgi_overrides.update({
            # Make sure uWSGI spawns an HTTP server for us as we don't
            # have a proxy/load-balancer in front in dev mode.
            "http": "%s:%s" % (host, port),
            "protocol": "uwsgi",
            # This is needed to prevent https://git.io/fj7Lw
            "uwsgi-socket": None,
        })

    if workers:
        if settings.CELERY_ALWAYS_EAGER:
            raise click.ClickException(
                "Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers."
            )

        daemons += [_get_daemon("worker"), _get_daemon("cron")]

        from sentry import eventstream

        if eventstream.requires_post_process_forwarder():
            daemons += [_get_daemon("post-process-forwarder")]

        if settings.SENTRY_DEV_PROCESS_SUBSCRIPTIONS:
            if not settings.SENTRY_EVENTSTREAM == "sentry.eventstream.kafka.KafkaEventStream":
                raise click.ClickException(
                    "`SENTRY_DEV_PROCESS_SUBSCRIPTIONS` can only be used when "
                    "`SENTRY_EVENTSTREAM=sentry.eventstream.kafka.KafkaEventStream`."
                )
            for name, topic in settings.KAFKA_SUBSCRIPTION_RESULT_TOPICS.items(
            ):
                daemons += [
                    _get_daemon("subscription-consumer",
                                "--topic",
                                topic,
                                suffix=name)
                ]

    if settings.SENTRY_USE_RELAY:
        daemons += [_get_daemon("ingest")]

    if needs_https and has_https:
        https_port = six.text_type(parsed_url.port)
        https_host = parsed_url.hostname

        # Determine a random port for the backend http server
        import socket

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        s.bind((host, 0))
        port = s.getsockname()[1]
        s.close()
        bind = "%s:%d" % (host, port)

        daemons += [("https", [
            "https", "-host", https_host, "-listen", host + ":" + https_port,
            bind
        ])]

    from sentry.runner.commands.devservices import _prepare_containers

    for name, container_options in _prepare_containers("sentry",
                                                       silent=True).items():
        if container_options.get("with_devserver", False):
            daemons += [(name,
                         ["sentry", "devservices", "attach", "--fast", name])]

    # A better log-format for local dev when running through honcho,
    # but if there aren't any other daemons, we don't want to override.
    if daemons:
        uwsgi_overrides[
            "log-format"] = "%(method) %(status) %(uri) %(proto) %(size)"
    else:
        uwsgi_overrides[
            "log-format"] = "[%(ltime)] %(method) %(status) %(uri) %(proto) %(size)"

    server = SentryHTTPServer(host=host,
                              port=port,
                              workers=1,
                              extra_options=uwsgi_overrides,
                              debug=debug_server)

    # If we don't need any other daemons, just launch a normal uwsgi webserver
    # and avoid dealing with subprocesses
    if not daemons:
        return server.run()

    import sys
    from subprocess import list2cmdline
    from honcho.manager import Manager
    from honcho.printer import Printer

    os.environ["PYTHONUNBUFFERED"] = "true"

    if debug_server:
        threading.Thread(target=server.run).start()
    else:
        # Make sure that the environment is prepared before honcho takes over
        # This sets all the appropriate uwsgi env vars, etc
        server.prepare_environment()
        daemons += [_get_daemon("server")]

    if styleguide:
        daemons += [_get_daemon("storybook")]

    cwd = os.path.realpath(
        os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))

    honcho_printer = Printer(prefix=prefix)

    if pretty:
        from sentry.runner.formatting import monkeypatch_honcho_write

        honcho_printer.write = types.MethodType(monkeypatch_honcho_write,
                                                honcho_printer)

    manager = Manager(honcho_printer)
    for name, cmd in daemons:
        manager.add_process(name, list2cmdline(cmd), quiet=False, cwd=cwd)

    manager.loop()
    sys.exit(manager.returncode)
Esempio n. 35
0
import sys

from honcho.manager import Manager

m = Manager()
m.add_process('ls', 'python manage.py runserver')
m.loop()

sys.exit(m.returncode)