def up(project, exclude):
"""
Run/update dependent services.
"""
os.environ["SENTRY_SKIP_BACKEND_VALIDATION"] = "1"
exclude = set(chain.from_iterable(x.split(",") for x in exclude))
from sentry.runner import configure
configure()
from django.conf import settings
client = get_docker_client()
get_or_create(client, "network", project)
containers = _prepare_containers(project)
for name, options in settings.SENTRY_DEVSERVICES.items():
if name in exclude:
continue
if name not in containers:
continue
_start_service(client, name, containers, project)
def up(ctx, services, project, exclude, fast):
"""
Run/update all devservices in the background.
The default is everything, however you may pass positional arguments to specify
an explicit list of services to bring up.
You may also exclude services, for example: --exclude redis --exclude postgres.
"""
from sentry.runner import configure
configure()
containers = _prepare_containers(project, silent=True)
selected_services = set()
if services:
for service in services:
if service not in containers:
click.secho(
f"Service `{service}` is not known or not enabled.\n",
err=True,
fg="red",
)
click.secho("Services that are available:\n" +
"\n".join(containers.keys()) + "\n",
err=True)
raise click.Abort()
selected_services.add(service)
else:
selected_services = set(containers.keys())
for service in exclude:
if service not in containers:
click.secho(f"Service `{service}` is not known or not enabled.\n",
err=True,
fg="red")
click.secho("Services that are available:\n" +
"\n".join(containers.keys()) + "\n",
err=True)
raise click.Abort()
selected_services.remove(service)
if fast:
click.secho(
"> Warning! Fast mode completely eschews any image updating, so services may be stale.",
err=True,
fg="red",
)
get_or_create(ctx.obj["client"], "network", project)
for name in selected_services:
_start_service(ctx.obj["client"],
ctx.obj["low_level_client"],
name,
containers,
project,
fast=fast)
def up(services, project, exclude, fast):
"""
Run/update dependent services.
The default is everything, however you may pass positional arguments to specify
an explicit list of services to bring up.
You may also exclude services, for example: --exclude redis --exclude postgres.
"""
os.environ["SENTRY_SKIP_BACKEND_VALIDATION"] = "1"
from sentry.runner import configure
configure()
containers = _prepare_containers(project, silent=True)
for service in exclude:
if service not in containers:
click.secho(
"Service `{}` is not known or not enabled.\n".format(service),
err=True,
fg="red")
click.secho("Services that are available:\n" +
"\n".join(containers.keys()) + "\n",
err=True)
raise click.Abort()
if services:
selected_containers = {}
for service in services:
if service not in containers:
click.secho(
"Service `{}` is not known or not enabled.\n".format(
service),
err=True,
fg="red",
)
click.secho("Services that are available:\n" +
"\n".join(containers.keys()) + "\n",
err=True)
raise click.Abort()
selected_containers[service] = containers[service]
containers = selected_containers
if fast:
click.secho(
"> Warning! Fast mode completely eschews any image updating, so services may be stale.",
err=True,
fg="red",
)
client = get_docker_client()
get_or_create(client, "network", project)
for name, container_options in containers.items():
if name in exclude:
continue
_start_service(client, name, containers, project, fast=fast)
def inner(ctx, *args, **kwargs):
# HACK: We can't call `configure()` from within tests
# since we don't load config files from disk, so we
# need a way to bypass this initialization step
if os.environ.get('_SENTRY_SKIP_CONFIGURATION') != '1':
from sentry.runner import configure
configure()
return ctx.invoke(f, *args, **kwargs)
def inner(ctx, *args, **kwargs):
# HACK: We can't call `configure()` from within tests
# since we don't load config files from disk, so we
# need a way to bypass this initialization step
if os.environ.get('_SENTRY_SKIP_CONFIGURATION') != '1':
from sentry.runner import configure
configure()
return ctx.invoke(f, *args, **kwargs)
def multiprocess_worker(task_queue):
# Configure within each Process
import logging
from sentry.utils.imports import import_string
logger = logging.getLogger('sentry.cleanup')
configured = False
while True:
j = task_queue.get()
if j == _STOP_WORKER:
task_queue.task_done()
return
# On first task, configure Sentry environment
if not configured:
from sentry.runner import configure
configure()
from sentry import models
from sentry import deletions
from sentry import similarity
skip_models = [
# Handled by other parts of cleanup
models.Event,
models.EventMapping,
models.EventAttachment,
models.UserReport,
models.Group,
models.GroupEmailThread,
models.GroupRuleStatus,
models.GroupHashTombstone,
# Handled by TTL
similarity.features,
] + [b[0] for b in EXTRA_BULK_QUERY_DELETES]
configured = True
model, chunk = j
model = import_string(model)
try:
task = deletions.get(
model=model,
query={'id__in': chunk},
skip_models=skip_models,
transaction_id=uuid4().hex,
)
while True:
if not task.chunk():
break
except Exception as e:
logger.exception(e)
finally:
task_queue.task_done()
def attach(project, fast, service):
"""
Run a single devservice in foreground, as opposed to `up` which runs all of
them in the background.
Accepts a single argument, the name of the service to spawn. The service
will run with output printed to your terminal, and the ability to kill it
with ^C. This is used in devserver.
Note: This does not update images, you will have to use `devservices up`
for that.
"""
os.environ["SENTRY_SKIP_BACKEND_VALIDATION"] = "1"
from sentry.runner import configure
configure()
client = get_docker_client()
containers = _prepare_containers(project, silent=True)
if service not in containers:
raise click.ClickException(
"Service `{}` is not known or not enabled.".format(service))
container = _start_service(client,
service,
containers,
project,
fast=fast,
always_start=True)
def exit_handler(*_):
try:
click.echo("Stopping {}".format(service))
container.stop()
click.echo("Removing {}".format(service))
container.remove()
except KeyboardInterrupt:
pass
signal.signal(signal.SIGINT, exit_handler)
signal.signal(signal.SIGTERM, exit_handler)
for line in container.logs(stream=True, since=int(time.time() - 20)):
click.echo(line, nl=False)
def attach(ctx, project, fast, service):
"""
Run a single devservice in the foreground.
Accepts a single argument, the name of the service to spawn. The service
will run with output printed to your terminal, and the ability to kill it
with ^C. This is used in devserver.
Note: This does not update images, you will have to use `devservices up`
for that.
"""
from sentry.runner import configure
configure()
containers = _prepare_containers(project, silent=True)
if service not in containers:
raise click.ClickException(
f"Service `{service}` is not known or not enabled.")
container = _start_service(
ctx.obj["client"],
ctx.obj["low_level_client"],
service,
containers,
project,
fast=fast,
always_start=True,
)
def exit_handler(*_):
try:
click.echo(f"Stopping {service}")
container.stop()
click.echo(f"Removing {service}")
container.remove()
except KeyboardInterrupt:
pass
signal.signal(signal.SIGINT, exit_handler)
signal.signal(signal.SIGTERM, exit_handler)
for line in container.logs(stream=True, since=int(time.time() - 20)):
click.echo(line, nl=False)
def attach(project, is_devserver, service):
"""
Run a single devservice in foreground, as opposed to `up` which runs all of
them in the background.
Accepts a single argument, the name of the service to spawn. The service
will run with output printed to your terminal, and the ability to kill it
with ^C. This is used in devserver.
Note: This does not update images, you will have to use `devservices up`
for that.
"""
os.environ["SENTRY_SKIP_BACKEND_VALIDATION"] = "1"
from sentry.runner import configure
configure()
client = get_docker_client()
containers = _prepare_containers(project)
container = _start_service(client,
service,
containers,
project,
devserver_override=is_devserver)
def exit_handler(*_):
click.echo("Shutting down {}".format(service))
try:
container.stop()
except KeyboardInterrupt:
pass
atexit.register(exit_handler)
signal.signal(signal.SIGINT, exit_handler)
signal.signal(signal.SIGTERM, exit_handler)
for line in container.logs(stream=True):
click.echo(line, nl=False)
def up(project, exclude, fast):
"""
Run/update dependent services.
"""
os.environ["SENTRY_SKIP_BACKEND_VALIDATION"] = "1"
exclude = set(chain.from_iterable(x.split(",") for x in exclude))
from sentry.runner import configure
configure()
from django.conf import settings
client = get_docker_client()
get_or_create(client, "network", project)
containers = _prepare_containers(project)
if fast:
click.secho(
"> Warning! Fast mode completely eschews any image updating, so services may be stale.",
err=True,
fg="red",
)
for name, options in settings.SENTRY_DEVSERVICES.items():
if name in exclude:
continue
if name not in containers:
continue
_start_service(client, name, containers, project, fast=fast)
def rm(project, services):
"""
Shut down and delete all services and associated data.
Useful if you'd like to start with a fresh slate.
The default is everything, however you may pass positional arguments to specify
an explicit list of services to remove.
"""
import docker
client = get_docker_client()
from sentry.runner import configure
configure()
containers = _prepare_containers(project, silent=True)
if services:
selected_containers = {}
for service in services:
# XXX: This code is also fairly duplicated in here at this point, so dedupe in the future.
if service not in containers:
click.secho(
"Service `{}` is not known or not enabled.\n".format(
service),
err=True,
fg="red",
)
click.secho("Services that are available:\n" +
"\n".join(containers.keys()) + "\n",
err=True)
raise click.Abort()
selected_containers[service] = containers[service]
containers = selected_containers
click.confirm(
"""
This will delete these services and all of their data:
%s
Are you sure you want to continue?""" % "\n".join(containers.keys()),
abort=True,
)
for service_name, container_options in containers.items():
try:
container = client.containers.get(container_options["name"])
except docker.errors.NotFound:
click.secho(
"> WARNING: non-existent container '%s'" %
container_options["name"],
err=True,
fg="yellow",
)
continue
click.secho("> Stopping '%s' container" % container_options["name"],
err=True,
fg="red")
container.stop()
click.secho("> Removing '%s' container" % container_options["name"],
err=True,
fg="red")
container.remove()
prefix = project + "_"
for volume in client.volumes.list():
if volume.name.startswith(prefix):
if not services or volume.name[len(prefix):] in services:
click.secho("> Removing '%s' volume" % volume.name,
err=True,
fg="red")
volume.remove()
if not services:
try:
network = client.networks.get(project)
except docker.errors.NotFound:
pass
else:
click.secho("> Removing '%s' network" % network.name,
err=True,
fg="red")
network.remove()
def up(project, exclude):
"Run/update dependent services."
os.environ['SENTRY_SKIP_BACKEND_VALIDATION'] = '1'
from sentry.runner import configure
configure()
from django.conf import settings
from sentry import options as sentry_options
import docker
client = get_docker_client()
# This is brittle, but is the best way now to limit what
# services are run if they're not needed.
if not exclude:
exclude = set()
if 'bigtable' not in settings.SENTRY_NODESTORE:
exclude |= {'bigtable'}
if 'memcached' not in settings.CACHES.get('default', {}).get('BACKEND'):
exclude |= {'memcached'}
if 'kafka' in settings.SENTRY_EVENTSTREAM:
pass
elif 'snuba' in settings.SENTRY_EVENTSTREAM:
click.secho(
'! Skipping kafka and zookeeper since your eventstream backend does not require it',
err=True,
fg='cyan')
exclude |= {'kafka', 'zookeeper'}
else:
click.secho(
'! Skipping kafka, zookeeper, snuba, and clickhouse since your eventstream backend does not require it',
err=True,
fg='cyan')
exclude |= {'kafka', 'zookeeper', 'snuba', 'clickhouse'}
if not sentry_options.get('symbolicator.enabled'):
exclude |= {'symbolicator'}
get_or_create(client, 'network', project)
containers = {}
for name, options in settings.SENTRY_DEVSERVICES.items():
if name in exclude:
continue
options = options.copy()
options['network'] = project
options['detach'] = True
options['name'] = project + '_' + name
options.setdefault('ports', {})
options.setdefault('environment', {})
options.setdefault('restart_policy', {'Name': 'on-failure'})
options['ports'] = ensure_interface(options['ports'])
containers[name] = options
pulled = set()
for name, options in containers.items():
# HACK(mattrobenolt): special handle snuba backend becuase it needs to
# handle different values based on the eventstream backend
# For snuba, we can't run the full suite of devserver, but can only
# run the api.
if name == 'snuba' and 'snuba' in settings.SENTRY_EVENTSTREAM:
options['environment'].pop('DEFAULT_BROKERS', None)
options['command'] = ['devserver', '--no-workers']
for key, value in options['environment'].items():
options['environment'][key] = value.format(containers=containers)
if options.pop('pull', False) and options['image'] not in pulled:
click.secho("> Pulling image '%s'" % options['image'], err=True, fg='green')
client.images.pull(options['image'])
pulled.add(options['image'])
for mount in options.get('volumes', {}).keys():
if '/' not in mount:
get_or_create(client, 'volume', project + '_' + mount)
options['volumes'][project + '_' + mount] = options['volumes'].pop(mount)
try:
container = client.containers.get(options['name'])
except docker.errors.NotFound:
pass
else:
container.stop()
container.remove()
listening = ''
if options['ports']:
listening = ' (listening: %s)' % ', '.join(map(text_type, options['ports'].values()))
click.secho("> Creating '%s' container%s" %
(options['name'], listening), err=True, fg='yellow')
client.containers.run(**options)
def up(ctx, services, project, exclude, fast, skip_only_if):
"""
Run/update all devservices in the background.
The default is everything, however you may pass positional arguments to specify
an explicit list of services to bring up.
You may also exclude services, for example: --exclude redis --exclude postgres.
"""
from sentry.runner import configure
configure()
containers = _prepare_containers(project,
skip_only_if=skip_only_if,
silent=True)
selected_services = set()
if services:
for service in services:
if service not in containers:
click.secho(
f"Service `{service}` is not known or not enabled.\n",
err=True,
fg="red",
)
click.secho("Services that are available:\n" +
"\n".join(containers.keys()) + "\n",
err=True)
raise click.Abort()
selected_services.add(service)
else:
selected_services = set(containers.keys())
for service in exclude:
if service not in containers:
click.secho(f"Service `{service}` is not known or not enabled.\n",
err=True,
fg="red")
click.secho("Services that are available:\n" +
"\n".join(containers.keys()) + "\n",
err=True)
raise click.Abort()
selected_services.remove(service)
if fast:
click.secho(
"> Warning! Fast mode completely eschews any image updating, so services may be stale.",
err=True,
fg="red",
)
get_or_create(ctx.obj["client"], "network", project)
with ThreadPoolExecutor(max_workers=len(selected_services)) as executor:
futures = []
for name in selected_services:
futures.append(
executor.submit(
_start_service,
ctx.obj["client"],
name,
containers,
project,
fast=fast,
))
for future in as_completed(futures):
# If there was an exception, reraising it here to the main thread
# will not terminate the whole python process. We'd like to report
# on this exception and stop as fast as possible, so terminate
# ourselves. I believe (without verification) that the OS is now
# free to cleanup these threads, but not sure if they'll remain running
# in the background. What matters most is that we regain control
# of the terminal.
e = future.exception()
if e:
click.echo(e)
me = os.getpid()
os.kill(me, signal.SIGTERM)
def _configure():
# Add the project to the python path
sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir))
# Disable backend validation
# os.environ['SENTRY_SKIP_BACKEND_VALIDATION'] = 1
configure()
def up(project, exclude):
"Run/update dependent services."
os.environ['SENTRY_SKIP_BACKEND_VALIDATION'] = '1'
exclude = set(chain.from_iterable(x.split(',') for x in exclude))
from sentry.runner import configure
configure()
from django.conf import settings
from sentry import options as sentry_options
import docker
client = get_docker_client()
# This is brittle, but is the best way now to limit what
# services are run if they're not needed.
if not exclude:
exclude = set()
if 'bigtable' not in settings.SENTRY_NODESTORE:
exclude |= {'bigtable'}
if 'memcached' not in settings.CACHES.get('default', {}).get('BACKEND'):
exclude |= {'memcached'}
if 'kafka' in settings.SENTRY_EVENTSTREAM:
pass
elif 'snuba' in settings.SENTRY_EVENTSTREAM:
click.secho(
'! Skipping kafka and zookeeper since your eventstream backend does not require it',
err=True,
fg='cyan')
exclude |= {'kafka', 'zookeeper'}
else:
click.secho(
'! Skipping kafka, zookeeper, snuba, and clickhouse since your eventstream backend does not require it',
err=True,
fg='cyan')
exclude |= {'kafka', 'zookeeper', 'snuba', 'clickhouse'}
if not sentry_options.get('symbolicator.enabled'):
exclude |= {'symbolicator'}
get_or_create(client, 'network', project)
containers = {}
for name, options in settings.SENTRY_DEVSERVICES.items():
if name in exclude:
continue
options = options.copy()
options['network'] = project
options['detach'] = True
options['name'] = project + '_' + name
options.setdefault('ports', {})
options.setdefault('environment', {})
options.setdefault('restart_policy', {'Name': 'on-failure'})
options['ports'] = ensure_interface(options['ports'])
containers[name] = options
pulled = set()
for name, options in containers.items():
# HACK(mattrobenolt): special handle snuba backend becuase it needs to
# handle different values based on the eventstream backend
# For snuba, we can't run the full suite of devserver, but can only
# run the api.
if name == 'snuba' and 'snuba' in settings.SENTRY_EVENTSTREAM:
options['environment'].pop('DEFAULT_BROKERS', None)
options['command'] = ['devserver', '--no-workers']
for key, value in options['environment'].items():
options['environment'][key] = value.format(containers=containers)
if options.pop('pull', False) and options['image'] not in pulled:
click.secho("> Pulling image '%s'" % options['image'], err=True, fg='green')
client.images.pull(options['image'])
pulled.add(options['image'])
for mount in options.get('volumes', {}).keys():
if '/' not in mount:
get_or_create(client, 'volume', project + '_' + mount)
options['volumes'][project + '_' + mount] = options['volumes'].pop(mount)
try:
container = client.containers.get(options['name'])
except docker.errors.NotFound:
pass
else:
container.stop()
container.remove()
listening = ''
if options['ports']:
listening = ' (listening: %s)' % ', '.join(map(text_type, options['ports'].values()))
click.secho("> Creating '%s' container%s" %
(options['name'], listening), err=True, fg='yellow')
client.containers.run(**options)
def cli(output_path, output_format):
global OUTPUT_PATH
if output_path is not None:
OUTPUT_PATH = os.path.abspath(output_path)
with apidoc_containers():
from sentry.runner import configure
configure()
sentry = Popen([
"sentry",
"--config=" + SENTRY_CONFIG,
"run",
"web",
"-w",
"1",
"--bind",
"127.0.0.1:9000",
])
from django.core.management import call_command
call_command(
"migrate",
interactive=False,
traceback=True,
verbosity=0,
migrate=True,
merge=True,
ignore_ghost_migrations=True,
)
utils = MockUtils()
report("org", "Creating user and organization")
user = utils.create_user("*****@*****.**")
org = utils.create_org("The Interstellar Jurisdiction", owner=user)
report("auth", "Creating api token")
api_token = utils.create_api_token(user)
report("org", "Creating team")
team = utils.create_team("Powerful Abolitionist", org=org)
utils.join_team(team, user)
projects = []
for project_name in "Pump Station", "Prime Mover":
report("project", 'Creating project "%s"' % project_name)
project = utils.create_project(project_name, teams=[team], org=org)
release = utils.create_release(project=project, user=user)
report("event", 'Creating event for "%s"' % project_name)
event1 = utils.create_event(project=project,
release=release,
platform="python")
event2 = utils.create_event(project=project,
release=release,
platform="java")
projects.append({
"project": project,
"release": release,
"events": [event1, event2]
})
# HACK: the scenario in ProjectDetailsEndpoint#put requires our integration docs to be in place
# so that we can validate the platform. We create the docker container that runs generator.py
# with SENTRY_LIGHT_BUILD=1, which doesn't run `sync_docs` and `sync_docs` requires sentry
# to be configured, which we do in this file. So, we need to do the sync_docs here.
sync_docs(quiet=True)
vars = {
"org": org,
"me": user,
"api_token": api_token,
"teams": [{
"team": team,
"projects": projects
}],
}
scenario_map = {}
report("docs", "Collecting scenarios")
for scenario_ident, func in iter_scenarios():
scenario = run_scenario(vars, scenario_ident, func)
scenario_map[scenario_ident] = scenario
section_mapping = {}
report("docs", "Collecting endpoint documentation")
for endpoint in iter_endpoints():
report("endpoint",
'Collecting docs for "%s"' % endpoint["endpoint_name"])
section_mapping.setdefault(endpoint["section"],
[]).append(endpoint)
sections = get_sections()
if output_format in ("json", "both"):
output_json(sections, scenario_map, section_mapping)
if output_format in ("markdown", "both"):
output_markdown(sections, scenario_map, section_mapping)
if sentry is not None:
report("sentry", "Shutting down sentry server")
sentry.kill()
sentry.wait()
def cleanup(days, project, concurrency, silent, model, router, timed):
"""Delete a portion of trailing data based on creation date.
All data that is older than `--days` will be deleted. The default for
this is 30 days. In the default setting all projects will be truncated
but if you have a specific project you want to limit this to this can be
done with the `--project` flag which accepts a project ID or a string
with the form `org/project` where both are slugs.
"""
if concurrency < 1:
click.echo('Error: Minimum concurrency is 1', err=True)
raise click.Abort()
os.environ['_SENTRY_CLEANUP'] = '1'
# Make sure we fork off multiprocessing pool
# before we import or configure the app
from multiprocessing import Process, JoinableQueue as Queue
pool = []
task_queue = Queue(1000)
for _ in xrange(concurrency):
p = Process(target=multiprocess_worker, args=(task_queue, ))
p.daemon = True
p.start()
pool.append(p)
from sentry.runner import configure
configure()
from django.db import router as db_router
from sentry.app import nodestore
from sentry.db.deletion import BulkDeleteQuery
from sentry import models
if timed:
import time
from sentry.utils import metrics
start_time = time.time()
# list of models which this query is restricted to
model_list = {m.lower() for m in model}
def is_filtered(model):
if router is not None and db_router.db_for_write(model) != router:
return True
if not model_list:
return False
return model.__name__.lower() not in model_list
# Deletions that use `BulkDeleteQuery` (and don't need to worry about child relations)
# (model, datetime_field, order_by)
BULK_QUERY_DELETES = [
(models.EventMapping, 'date_added', '-date_added'),
(models.EventAttachment, 'date_added', None),
(models.UserReport, 'date_added', None),
(models.GroupEmailThread, 'date', None),
(models.GroupRuleStatus, 'date_added', None),
] + EXTRA_BULK_QUERY_DELETES
# Deletions that use the `deletions` code path (which handles their child relations)
# (model, datetime_field, order_by)
DELETES = (
(models.Event, 'datetime', 'datetime'),
(models.Group, 'last_seen', 'last_seen'),
)
if not silent:
click.echo('Removing expired values for LostPasswordHash')
if is_filtered(models.LostPasswordHash):
if not silent:
click.echo('>> Skipping LostPasswordHash')
else:
models.LostPasswordHash.objects.filter(date_added__lte=timezone.now() -
timedelta(hours=48)).delete()
if is_filtered(models.OrganizationMember) and not silent:
click.echo('>> Skipping OrganizationMember')
else:
if not silent:
click.echo('Removing expired values for OrganizationMember')
expired_threshold = timezone.now() - timedelta(days=days)
models.OrganizationMember.delete_expired(expired_threshold)
for model in [models.ApiGrant, models.ApiToken]:
if not silent:
click.echo(u'Removing expired values for {}'.format(
model.__name__))
if is_filtered(model):
if not silent:
click.echo(u'>> Skipping {}'.format(model.__name__))
else:
model.objects.filter(expires_at__lt=(
timezone.now() -
timedelta(days=API_TOKEN_TTL_IN_DAYS)), ).delete()
project_id = None
if project:
click.echo(
"Bulk NodeStore deletion not available for project selection",
err=True)
project_id = get_project(project)
if project_id is None:
click.echo('Error: Project not found', err=True)
raise click.Abort()
else:
if not silent:
click.echo("Removing old NodeStore values")
cutoff = timezone.now() - timedelta(days=days)
try:
nodestore.cleanup(cutoff)
except NotImplementedError:
click.echo("NodeStore backend does not support cleanup operation",
err=True)
for bqd in BULK_QUERY_DELETES:
if len(bqd) == 4:
model, dtfield, order_by, chunk_size = bqd
else:
chunk_size = 10000
model, dtfield, order_by = bqd
if not silent:
click.echo(
u"Removing {model} for days={days} project={project}".format(
model=model.__name__,
days=days,
project=project or '*',
))
if is_filtered(model):
if not silent:
click.echo('>> Skipping %s' % model.__name__)
else:
BulkDeleteQuery(
model=model,
dtfield=dtfield,
days=days,
project_id=project_id,
order_by=order_by,
).execute(chunk_size=chunk_size)
for model, dtfield, order_by in DELETES:
if not silent:
click.echo(
u"Removing {model} for days={days} project={project}".format(
model=model.__name__,
days=days,
project=project or '*',
))
if is_filtered(model):
if not silent:
click.echo('>> Skipping %s' % model.__name__)
else:
imp = '.'.join((model.__module__, model.__name__))
q = BulkDeleteQuery(
model=model,
dtfield=dtfield,
days=days,
project_id=project_id,
order_by=order_by,
)
for chunk in q.iterator(chunk_size=100):
task_queue.put((imp, chunk))
task_queue.join()
# Clean up FileBlob instances which are no longer used and aren't super
# recent (as there could be a race between blob creation and reference)
if not silent:
click.echo("Cleaning up unused FileBlob references")
if is_filtered(models.FileBlob):
if not silent:
click.echo('>> Skipping FileBlob')
else:
cleanup_unused_files(silent)
# Shut down our pool
for _ in pool:
task_queue.put(_STOP_WORKER)
# And wait for it to drain
for p in pool:
p.join()
if timed:
duration = int(time.time() - start_time)
metrics.timing('cleanup.duration',
duration,
instance=router,
sample_rate=1.0)
click.echo("Clean up took %s second(s)." % duration)
def cleanup(days, project, concurrency, silent, model, router, timed):
"""Delete a portion of trailing data based on creation date.
All data that is older than `--days` will be deleted. The default for
this is 30 days. In the default setting all projects will be truncated
but if you have a specific project you want to limit this to this can be
done with the `--project` flag which accepts a project ID or a string
with the form `org/project` where both are slugs.
"""
if concurrency < 1:
click.echo('Error: Minimum concurrency is 1', err=True)
raise click.Abort()
os.environ['_SENTRY_CLEANUP'] = '1'
# Make sure we fork off multiprocessing pool
# before we import or configure the app
from multiprocessing import Process, JoinableQueue as Queue
pool = []
task_queue = Queue(1000)
for _ in xrange(concurrency):
p = Process(target=multiprocess_worker, args=(task_queue,))
p.daemon = True
p.start()
pool.append(p)
from sentry.runner import configure
configure()
from django.db import router as db_router
from sentry.app import nodestore
from sentry.db.deletion import BulkDeleteQuery
from sentry import models
if timed:
import time
from sentry.utils import metrics
start_time = time.time()
# list of models which this query is restricted to
model_list = {m.lower() for m in model}
def is_filtered(model):
if router is not None and db_router.db_for_write(model) != router:
return True
if not model_list:
return False
return model.__name__.lower() not in model_list
# Deletions that use `BulkDeleteQuery` (and don't need to worry about child relations)
# (model, datetime_field, order_by)
BULK_QUERY_DELETES = [
(models.EventMapping, 'date_added', '-date_added'),
(models.EventAttachment, 'date_added', None),
(models.UserReport, 'date_added', None),
(models.GroupEmailThread, 'date', None),
(models.GroupRuleStatus, 'date_added', None),
] + EXTRA_BULK_QUERY_DELETES
# Deletions that use the `deletions` code path (which handles their child relations)
# (model, datetime_field, order_by)
DELETES = (
(models.Event, 'datetime', 'datetime'),
(models.Group, 'last_seen', 'last_seen'),
)
if not silent:
click.echo('Removing expired values for LostPasswordHash')
if is_filtered(models.LostPasswordHash):
if not silent:
click.echo('>> Skipping LostPasswordHash')
else:
models.LostPasswordHash.objects.filter(
date_added__lte=timezone.now() - timedelta(hours=48)
).delete()
if is_filtered(models.OrganizationMember) and not silent:
click.echo('>> Skipping OrganizationMember')
else:
click.echo('Removing expired values for OrganizationMember')
expired_threshold = timezone.now() - timedelta(days=days)
models.OrganizationMember.delete_expired(expired_threshold)
for model in [models.ApiGrant, models.ApiToken]:
if not silent:
click.echo(u'Removing expired values for {}'.format(model.__name__))
if is_filtered(model):
if not silent:
click.echo(u'>> Skipping {}'.format(model.__name__))
else:
model.objects.filter(
expires_at__lt=(timezone.now() - timedelta(days=days)),
).delete()
project_id = None
if project:
click.echo(
"Bulk NodeStore deletion not available for project selection", err=True)
project_id = get_project(project)
if project_id is None:
click.echo('Error: Project not found', err=True)
raise click.Abort()
else:
if not silent:
click.echo("Removing old NodeStore values")
cutoff = timezone.now() - timedelta(days=days)
try:
nodestore.cleanup(cutoff)
except NotImplementedError:
click.echo(
"NodeStore backend does not support cleanup operation", err=True)
for bqd in BULK_QUERY_DELETES:
if len(bqd) == 4:
model, dtfield, order_by, chunk_size = bqd
else:
chunk_size = 10000
model, dtfield, order_by = bqd
if not silent:
click.echo(
u"Removing {model} for days={days} project={project}".format(
model=model.__name__,
days=days,
project=project or '*',
)
)
if is_filtered(model):
if not silent:
click.echo('>> Skipping %s' % model.__name__)
else:
BulkDeleteQuery(
model=model,
dtfield=dtfield,
days=days,
project_id=project_id,
order_by=order_by,
).execute(chunk_size=chunk_size)
for model, dtfield, order_by in DELETES:
if not silent:
click.echo(
u"Removing {model} for days={days} project={project}".format(
model=model.__name__,
days=days,
project=project or '*',
)
)
if is_filtered(model):
if not silent:
click.echo('>> Skipping %s' % model.__name__)
else:
imp = '.'.join((model.__module__, model.__name__))
q = BulkDeleteQuery(
model=model,
dtfield=dtfield,
days=days,
project_id=project_id,
order_by=order_by,
)
for chunk in q.iterator(chunk_size=100):
task_queue.put((imp, chunk))
task_queue.join()
# Clean up FileBlob instances which are no longer used and aren't super
# recent (as there could be a race between blob creation and reference)
if not silent:
click.echo("Cleaning up unused FileBlob references")
if is_filtered(models.FileBlob):
if not silent:
click.echo('>> Skipping FileBlob')
else:
cleanup_unused_files(silent)
# Shut down our pool
for _ in pool:
task_queue.put(_STOP_WORKER)
# And wait for it to drain
for p in pool:
p.join()
if timed:
duration = int(time.time() - start_time)
metrics.timing('cleanup.duration', duration, instance=router)
click.echo("Clean up took %s second(s)." % duration)
import click
import urlparse
import logging
from datetime import datetime
from subprocess import Popen, PIPE
from contextlib import contextmanager
HERE = os.path.abspath(os.path.dirname(__file__))
SENTRY_CONFIG = os.environ['SENTRY_CONF'] = os.path.join(
HERE, 'sentry.conf.py')
os.environ['SENTRY_SKIP_BACKEND_VALIDATION'] = '1'
# No sentry or django imports before that point
from sentry.runner import configure
configure()
from django.conf import settings
# Fair game from here
from django.core.management import call_command
from sentry.utils.apidocs import Runner, MockUtils, iter_scenarios, \
iter_endpoints, get_sections
OUTPUT_PATH = os.path.join(HERE, 'cache')
HOST = urlparse.urlparse(settings.SENTRY_OPTIONS['system.url-prefix']).netloc
# We don't care about you, go away
_logger = logging.getLogger('sentry.events')
_logger.disabled = True
def devserver(reload, watchers, workers, browser_reload, styleguide, prefix,
environment, bind, vscode_debug, search_index):
"Starts a lightweight web server for development."
if ':' in bind:
host, port = bind.split(':', 1)
port = int(port)
else:
host = bind
port = None
import os
os.environ['CLIMS_ENVIRONMENT'] = environment
from sentry.runner import configure
configure()
from django.conf import settings
from sentry import options
from sentry.services.http import SentryHTTPServer
# NOTE: We have to set this variable here rather than directly in the config file
# because the config file is imported before we're able to set the _ENVIRONMENT variable
settings.DEBUG = environment == 'development'
url_prefix = options.get('system.url-prefix', '')
parsed_url = urlparse(url_prefix)
# Make sure we're trying to use a port that we can actually bind to
needs_https = (parsed_url.scheme == 'https'
and (parsed_url.port or 443) > 1024)
has_https = False
if needs_https:
from subprocess import check_output
try:
check_output(['which', 'https'])
has_https = True
except Exception:
has_https = False
from sentry.runner.initializer import show_big_error
show_big_error([
'missing `https` on your `$PATH`, but https is needed',
'`$ brew install mattrobenolt/stuff/https`',
])
uwsgi_overrides = {
# Make sure we don't try and use uwsgi protocol
'protocol': 'http',
# Make sure we reload really quickly for local dev in case it
# doesn't want to shut down nicely on it's own, NO MERCY
'worker-reload-mercy': 2,
# We need stdin to support pdb in devserver
'honour-stdin': True,
# accept ridiculously large files
'limit-post': 1 << 30,
# do something with chunked
'http-chunked-input': True,
}
if reload:
uwsgi_overrides['py-autoreload'] = 1
daemons = []
if watchers and not browser_reload:
daemons += settings.SENTRY_WATCHERS
# For javascript dev, if browser reload and watchers, then:
# devserver listen on PORT + 1
# webpack dev server listen on PORT + 2
# proxy listen on PORT
if watchers and browser_reload:
new_port = port + 1
os.environ['WEBPACK_DEV_PROXY'] = '%s' % port
os.environ['WEBPACK_DEV_PORT'] = '%s' % (new_port + 1)
os.environ['SENTRY_DEVSERVER_PORT'] = '%s' % new_port
port = new_port
daemons += [('jsproxy', ['yarn', 'dev-proxy']),
('webpack', ['yarn', 'dev-server'])]
if workers:
if settings.CELERY_ALWAYS_EAGER:
raise click.ClickException(
'Disable CELERY_ALWAYS_EAGER in your settings file to spawn workers.'
)
daemons += [
('worker', ['lims', 'run', 'worker', '-c', '1', '--autoreload']),
('cron', ['lims', 'run', 'cron', '--autoreload']),
]
if needs_https and has_https:
https_port = six.text_type(parsed_url.port)
https_host = parsed_url.hostname
# Determine a random port for the backend http server
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host, 0))
port = s.getsockname()[1]
s.close()
bind = '%s:%d' % (host, port)
daemons += [
('https', [
'https', '-host', https_host, '-listen',
host + ':' + https_port, bind
]),
]
# A better log-format for local dev when running through honcho,
# but if there aren't any other daemons, we don't want to override.
if daemons:
uwsgi_overrides[
'log-format'] = '"%(method) %(uri) %(proto)" %(status) %(size)'
else:
uwsgi_overrides[
'log-format'] = '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size)'
server = SentryHTTPServer(host=host,
port=port,
workers=1,
extra_options=uwsgi_overrides)
# If using vscode_debug, currently we're only starting the builtin django server:
if vscode_debug:
from django.core.management import execute_from_command_line
execute_from_command_line(
['', 'runserver', '--noreload', '--nothreading'])
return
# If we don't need any other daemons, just launch a normal uwsgi webserver
# and avoid dealing with subprocesses
if not daemons:
return server.run()
import sys
from subprocess import list2cmdline
from honcho.manager import Manager
from honcho.printer import Printer
os.environ['PYTHONUNBUFFERED'] = 'true'
# Make sure that the environment is prepared before honcho takes over
# This sets all the appropriate uwsgi env vars, etc
server.prepare_environment()
daemons += [
('server', ['lims', 'run', 'web']),
]
if styleguide:
daemons += [('storybook', ['yarn', 'storybook'])]
if search_index and False:
daemons += [('search_index', ['lims', 'run', 'search_index'])]
cwd = os.path.realpath(
os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))
manager = Manager(Printer(prefix=prefix))
for name, cmd in daemons:
manager.add_process(
name,
list2cmdline(cmd),
quiet=False,
cwd=cwd,
)
manager.loop()
sys.exit(manager.returncode)
"""
from __future__ import absolute_import
import os
import os.path
import sys
# Add the project to the python path
sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir))
# Configure the application only if it seemingly isnt already configured
from django.conf import settings
if not settings.configured:
from sentry.runner import configure
configure()
if settings.SESSION_FILE_PATH and not os.path.exists(settings.SESSION_FILE_PATH):
try:
os.makedirs(settings.SESSION_FILE_PATH)
except OSError:
pass
from django.core.handlers.wsgi import WSGIHandler
class FileWrapperWSGIHandler(WSGIHandler):
"""A WSGIHandler implementation that handles a StreamingHttpResponse
from django to leverage wsgi.file_wrapper for delivering large streaming
responses.
def initializer() -> None:
from sentry.runner import configure
configure()
def cli(ctx):
from sentry.runner import configure
configure()
def up(project, exclude):
"Run/update dependent services."
os.environ["SENTRY_SKIP_BACKEND_VALIDATION"] = "1"
exclude = set(chain.from_iterable(x.split(",") for x in exclude))
from sentry.runner import configure
configure()
from django.conf import settings
from sentry import options as sentry_options
import docker
client = get_docker_client()
get_or_create(client, "network", project)
containers = {}
for name, options in settings.SENTRY_DEVSERVICES.items():
if name in exclude:
continue
options = options.copy()
test_fn = options.pop("only_if", None)
if test_fn and not test_fn(settings, sentry_options):
click.secho("! Skipping {} due to only_if condition".format(name),
err=True,
fg="cyan")
continue
options["network"] = project
options["detach"] = True
options["name"] = project + "_" + name
options.setdefault("ports", {})
options.setdefault("environment", {})
options.setdefault("restart_policy", {"Name": "on-failure"})
options["ports"] = ensure_interface(options["ports"])
containers[name] = options
pulled = set()
for name, options in containers.items():
# HACK(mattrobenolt): special handle snuba backend because it needs to
# handle different values based on the eventstream backend
# For snuba, we can't run the full suite of devserver, but can only
# run the api.
if name == "snuba" and "snuba" in settings.SENTRY_EVENTSTREAM:
options["environment"].pop("DEFAULT_BROKERS", None)
options["command"] = ["devserver", "--no-workers"]
for key, value in options["environment"].items():
options["environment"][key] = value.format(containers=containers)
if options.pop("pull", False) and options["image"] not in pulled:
click.secho("> Pulling image '%s'" % options["image"],
err=True,
fg="green")
client.images.pull(options["image"])
pulled.add(options["image"])
for mount in options.get("volumes", {}).keys():
if "/" not in mount:
get_or_create(client, "volume", project + "_" + mount)
options["volumes"][project + "_" +
mount] = options["volumes"].pop(mount)
try:
container = client.containers.get(options["name"])
except docker.errors.NotFound:
pass
else:
container.stop()
container.remove()
listening = ""
if options["ports"]:
listening = " (listening: %s)" % ", ".join(
map(text_type, options["ports"].values()))
click.secho("> Creating '%s' container%s" %
(options["name"], listening),
err=True,
fg="yellow")
client.containers.run(**options)
def up(project, exclude):
"Run/update dependent services."
os.environ["SENTRY_SKIP_BACKEND_VALIDATION"] = "1"
exclude = set(chain.from_iterable(x.split(",") for x in exclude))
from sentry.runner import configure
configure()
from django.conf import settings
from sentry import options as sentry_options
import docker
client = get_docker_client()
# This is brittle, but is the best way now to limit what
# services are run if they're not needed.
if not exclude:
exclude = set()
if "bigtable" not in settings.SENTRY_NODESTORE:
exclude |= {"bigtable"}
if "memcached" not in settings.CACHES.get("default", {}).get("BACKEND"):
exclude |= {"memcached"}
if "kafka" in settings.SENTRY_EVENTSTREAM:
pass
elif "snuba" in settings.SENTRY_EVENTSTREAM:
click.secho(
"! Skipping kafka and zookeeper since your eventstream backend does not require it",
err=True,
fg="cyan",
)
exclude |= {"kafka", "zookeeper"}
else:
click.secho(
"! Skipping kafka, zookeeper, snuba, and clickhouse since your eventstream backend does not require it",
err=True,
fg="cyan",
)
exclude |= {"kafka", "zookeeper", "snuba", "clickhouse"}
if not sentry_options.get("symbolicator.enabled"):
exclude |= {"symbolicator"}
get_or_create(client, "network", project)
containers = {}
for name, options in settings.SENTRY_DEVSERVICES.items():
if name in exclude:
continue
options = options.copy()
options["network"] = project
options["detach"] = True
options["name"] = project + "_" + name
options.setdefault("ports", {})
options.setdefault("environment", {})
options.setdefault("restart_policy", {"Name": "on-failure"})
options["ports"] = ensure_interface(options["ports"])
containers[name] = options
pulled = set()
for name, options in containers.items():
# HACK(mattrobenolt): special handle snuba backend because it needs to
# handle different values based on the eventstream backend
# For snuba, we can't run the full suite of devserver, but can only
# run the api.
if name == "snuba" and "snuba" in settings.SENTRY_EVENTSTREAM:
options["environment"].pop("DEFAULT_BROKERS", None)
options["command"] = ["devserver", "--no-workers"]
for key, value in options["environment"].items():
options["environment"][key] = value.format(containers=containers)
if options.pop("pull", False) and options["image"] not in pulled:
click.secho("> Pulling image '%s'" % options["image"],
err=True,
fg="green")
client.images.pull(options["image"])
pulled.add(options["image"])
for mount in options.get("volumes", {}).keys():
if "/" not in mount:
get_or_create(client, "volume", project + "_" + mount)
options["volumes"][project + "_" +
mount] = options["volumes"].pop(mount)
try:
container = client.containers.get(options["name"])
except docker.errors.NotFound:
pass
else:
container.stop()
container.remove()
listening = ""
if options["ports"]:
listening = " (listening: %s)" % ", ".join(
map(text_type, options["ports"].values()))
click.secho("> Creating '%s' container%s" %
(options["name"], listening),
err=True,
fg="yellow")
client.containers.run(**options)
