def test_get_db_engine(self):
from sentry.utils.db import get_db_engine
_databases = getattr(django_settings, 'DATABASES', {}).copy()
django_settings.DATABASES['default'] = {'ENGINE': 'blah.sqlite3'}
self.assertEquals(get_db_engine(), 'sqlite3')
django_settings.DATABASES['default'] = {'ENGINE': 'blah.mysql'}
self.assertEquals(get_db_engine(), 'mysql')
django_settings.DATABASES = _databases
def get_priority_sort_expression(model):
engine = get_db_engine(router.db_for_read(model))
table = get_sql_table(model)
if 'postgres' in engine:
return u'log({table}.times_seen) * 600 + {table}.last_seen::abstime::int'.format(table=table)
else:
# TODO: This should be improved on other databases where possible.
# (This doesn't work on some databases: SQLite for example doesn't
# have a built-in logarithm function.)
return u'{}.times_seen'.format(table)
def get_sql_date_trunc(col, db='default'):
conn = connections[db]
engine = get_db_engine(db)
# TODO: does extract work for sqlite?
if engine.startswith('oracle'):
method = conn.ops.date_trunc_sql('hh24', col)
else:
method = conn.ops.date_trunc_sql('hour', col)
return method
def evaluate(self, node, qn, connection):
engine = get_db_engine(getattr(connection, 'alias', 'default'))
if engine.startswith('postgresql'):
sql = 'log(times_seen) * 600 + last_seen::abstime::int'
elif engine.startswith('mysql'):
sql = 'log(times_seen) * 600 + unix_timestamp(last_seen)'
else:
# XXX: if we cant do it atomically let's do it the best we can
sql = int(self)
return (sql, [])
def as_sql(self, compiler, connection, function=None, template=None):
engine = get_db_engine(getattr(connection, 'alias', 'default'))
if engine.startswith('postgresql'):
sql = 'log(times_seen) * 600 + last_seen::abstime::int'
elif engine.startswith('mysql'):
sql = 'log(times_seen) * 600 + unix_timestamp(last_seen)'
else:
# XXX: if we cant do it atomically let's do it the best we can
sql = int(self)
return (sql, [])
def get_sql_date_trunc(col, db='default', grouper='hour'):
conn = connections[db]
engine = get_db_engine(db)
# TODO: does extract work for sqlite?
if engine.startswith('oracle'):
method = DATE_TRUNC_GROUPERS['oracle'].get(grouper, DATE_TRUNC_GROUPERS['default'][grouper])
else:
method = DATE_TRUNC_GROUPERS['default'][grouper]
return conn.ops.date_trunc_sql(method, col)
def evaluate(self, node, qn, connection):
engine = get_db_engine(getattr(connection, "alias", "default"))
if engine.startswith("postgresql"):
sql = "log(times_seen) * 600 + last_seen::abstime::int"
elif engine.startswith("mysql"):
sql = "log(times_seen) * 600 + unix_timestamp(last_seen)"
else:
# XXX: if we cant do it atomicly let's do it the best we can
sql = self.group.get_score()
return (sql, [])
def get_sort_clause(sort_by):
engine = get_db_engine('default')
if engine.startswith('sqlite'):
return SQLITE_SORT_CLAUSES[sort_by]
elif engine.startswith('mysql'):
return MYSQL_SORT_CLAUSES[sort_by]
elif engine.startswith('oracle'):
return ORACLE_SORT_CLAUSES[sort_by]
elif engine in MSSQL_ENGINES:
return MSSQL_SORT_CLAUSES[sort_by]
else:
return SORT_CLAUSES[sort_by]
def get_sql_date_trunc(col, db="default", grouper="hour"):
conn = connections[db]
engine = get_db_engine(db)
# TODO: does extract work for sqlite?
if engine.startswith("oracle"):
method = DATE_TRUNC_GROUPERS["oracle"].get(grouper, DATE_TRUNC_GROUPERS["default"][grouper])
if '"' not in col:
col = '"%s"' % col.upper()
else:
method = DATE_TRUNC_GROUPERS["default"][grouper]
return conn.ops.date_trunc_sql(method, col)
def get_accelerated(self, queryset=None, minutes=15):
# mintues should
from sentry.models import MessageCountByMinute
mcbm_tbl = MessageCountByMinute._meta.db_table
if queryset is None:
queryset = self
assert minutes >= settings.MINUTE_NORMALIZATION
engine = get_db_engine(queryset.db)
if engine.startswith('mysql'):
minute_clause = "interval %s minute"
else:
minute_clause = "interval '%s minutes'"
queryset = queryset.extra(
where=["%s.date >= now() - %s" % (mcbm_tbl, minute_clause % (minutes, ))],
).annotate(x=Sum('messagecountbyminute__times_seen')).order_by('id')
sql, params = queryset.query.get_compiler(queryset.db).as_sql()
before_select, after_select = str(sql).split('SELECT ', 1)
before_where, after_where = after_select.split(' WHERE ', 1)
before_group, after_group = after_where.split(' GROUP BY ', 1)
# Ensure we remove any ordering clause
after_group = after_group.split(' ORDER BY ')[0]
query = """
SELECT (SUM(%(mcbm_tbl)s.times_seen) + 1.0) / (COALESCE(z.accel, 0) + 1.0) as accel,
z.accel as prev_accel,
%(before_where)s
LEFT JOIN (SELECT a.group_id, SUM(a.times_seen) / 3.0 as accel
FROM %(mcbm_tbl)s as a
WHERE a.date BETWEEN now() - %(min_time)s
AND now() - %(min_time)s
GROUP BY a.group_id) as z
ON z.group_id = %(mcbm_tbl)s.group_id
WHERE %(before_group)s
GROUP BY prev_accel, %(after_group)s
HAVING SUM(%(mcbm_tbl)s.times_seen) > 0
ORDER BY accel DESC
""" % dict(
mcbm_tbl=mcbm_tbl,
before_where=before_where,
before_group=before_group,
after_group=after_group,
min_time=minute_clause % (minutes + 1,),
max_time=minute_clause % (minutes * 4,),
)
return RawQuerySet(self, query, params)
def get(self, request, team):
"""
Return a list of the newest groups for a given team.
The resulting query will find groups which have been seen since the
cutoff date, and then sort those by score, returning the highest scoring
groups first.
"""
minutes = int(request.REQUEST.get('minutes', 15))
limit = min(100, int(request.REQUEST.get('limit', 10)))
project_list = Project.objects.get_for_user(user=request.user, team=team)
project_dict = dict((p.id, p) for p in project_list)
cutoff = timedelta(minutes=minutes)
cutoff_dt = timezone.now() - cutoff
if get_db_engine('default') == 'sqlite':
sort_value = 'times_seen'
else:
sort_value = 'score'
group_list = list(
Group.objects.filter(
project__in=project_dict.keys(),
status=GroupStatus.UNRESOLVED,
active_at__gte=cutoff_dt,
).extra(
select={'sort_value': sort_value},
).order_by('-{}'.format(sort_value), '-first_seen')[:limit]
)
for group in group_list:
group._project_cache = project_dict.get(group.project_id)
return Response(
serialize(
group_list,
request.user,
GroupSerializer(
environment_func=self._get_environment_func(request, team.organization_id)
)
)
)
def get_group_tags(request, team, project, group_id, tag_name):
# XXX(dcramer): Consider this API deprecated as soon as it was implemented
cutoff = timezone.now() - timedelta(days=7)
engine = get_db_engine('default')
if 'postgres' in engine:
# This doesnt guarantee percentage is accurate, but it does ensure
# that the query has a maximum cost
cursor = connections['default'].cursor()
cursor.execute("""
SELECT SUM(t)
FROM (
SELECT times_seen as t
FROM sentry_messagefiltervalue
WHERE group_id = %s
AND key = %s
AND last_seen > NOW() - INTERVAL '7 days'
LIMIT 10000
) as a
""", [group_id, tag_name])
total = cursor.fetchone()[0] or 0
else:
total = GroupTagValue.objects.filter(
group=group_id,
key=tag_name,
last_seen__gte=cutoff,
).aggregate(t=Sum('times_seen'))['t'] or 0
unique_tags = GroupTagValue.objects.filter(
group=group_id,
key=tag_name,
last_seen__gte=cutoff,
).values_list('value', 'times_seen').order_by('-times_seen')[:10]
return json.dumps({
'name': tag_name,
'values': list(unique_tags),
'total': total,
})
event_list = event_list.filter(last_seen__gte=date_from)
elif date_to:
event_list = event_list.filter(last_seen__lte=date_to)
sort = request.GET.get("sort") or request.session.get("streamsort")
if sort not in SORT_OPTIONS:
sort = DEFAULT_SORT_OPTION
# Save last sort in session
if sort != request.session.get("streamsort"):
request.session["streamsort"] = sort
if sort.startswith("accel_") and not has_trending():
sort = DEFAULT_SORT_OPTION
engine = get_db_engine("default")
if engine.startswith("sqlite"):
score_clause = SQLITE_SORT_CLAUSES.get(sort)
filter_clause = SQLITE_SCORE_CLAUSES.get(sort)
elif engine.startswith("mysql"):
score_clause = MYSQL_SORT_CLAUSES.get(sort)
filter_clause = MYSQL_SCORE_CLAUSES.get(sort)
elif engine.startswith("oracle"):
score_clause = ORACLE_SORT_CLAUSES.get(sort)
filter_clause = ORACLE_SCORE_CLAUSES.get(sort)
elif engine in ("django_pytds", "sqlserver_ado", "sql_server.pyodbc"):
score_clause = MSSQL_SORT_CLAUSES.get(sort)
filter_clause = MSSQL_SCORE_CLAUSES.get(sort)
else:
score_clause = SORT_CLAUSES.get(sort)
filter_clause = SCORE_CLAUSES.get(sort)
def query(self, project, query=None, status=None, tags=None,
bookmarked_by=None, sort_by='date', date_filter='last_seen',
date_from=None, date_to=None, offset=0, limit=100):
from sentry.models import Group
queryset = Group.objects.filter(project=project)
if query:
# TODO(dcramer): if we want to continue to support search on SQL
# we should at least optimize this in Postgres so that it does
# the query filter **after** the index filters, and restricts the
# result set
queryset = queryset.filter(
Q(message__icontains=query) |
Q(culprit__icontains=query)
)
if status is not None:
queryset = queryset.filter(status=status)
if bookmarked_by:
queryset = queryset.filter(
bookmark_set__project=project,
bookmark_set__user=bookmarked_by,
)
if tags:
for k, v in tags.iteritems():
queryset = queryset.filter(**dict(
grouptag__key=k,
grouptag__value=v,
))
if date_filter == 'first_seen':
if date_from and date_to:
queryset = queryset.filter(
first_seen__gte=date_from,
first_seen__lte=date_to,
)
elif date_from:
queryset = queryset.filter(first_seen__gte=date_from)
elif date_to:
queryset = queryset.filter(first_seen__lte=date_to)
elif date_filter == 'last_seen':
if date_from and date_to:
queryset = queryset.filter(
first_seen__gte=date_from,
last_seen__lte=date_to,
)
elif date_from:
queryset = queryset.filter(last_seen__gte=date_from)
elif date_to:
queryset = queryset.filter(last_seen__lte=date_to)
engine = get_db_engine('default')
if engine.startswith('sqlite'):
score_clause = SQLITE_SORT_CLAUSES[sort_by]
filter_clause = SQLITE_SCORE_CLAUSES[sort_by]
elif engine.startswith('mysql'):
score_clause = MYSQL_SORT_CLAUSES[sort_by]
filter_clause = MYSQL_SCORE_CLAUSES[sort_by]
elif engine.startswith('oracle'):
score_clause = ORACLE_SORT_CLAUSES[sort_by]
filter_clause = ORACLE_SCORE_CLAUSES[sort_by]
elif engine in MSSQL_ENGINES:
score_clause = MSSQL_SORT_CLAUSES[sort_by]
filter_clause = MSSQL_SCORE_CLAUSES[sort_by]
else:
score_clause = SORT_CLAUSES[sort_by]
filter_clause = SCORE_CLAUSES[sort_by]
if sort_by == 'tottime':
queryset = queryset.filter(time_spent_count__gt=0)
elif sort_by == 'avgtime':
queryset = queryset.filter(time_spent_count__gt=0)
queryset = queryset.extra(
select={'sort_value': score_clause},
)
# HACK: don't sort by the same column twice
if sort_by == 'date':
queryset = queryset.order_by('-last_seen')
else:
queryset = queryset.order_by('-sort_value', '-last_seen')
return SearchResult(instances=list(queryset[offset:offset + limit]))
def _build_queryset(
self,
project,
query=None,
status=None,
tags=None,
bookmarked_by=None,
assigned_to=None,
first_release=None,
sort_by='date',
unassigned=None,
subscribed_by=None,
age_from=None,
age_from_inclusive=True,
age_to=None,
age_to_inclusive=True,
last_seen_from=None,
last_seen_from_inclusive=True,
last_seen_to=None,
last_seen_to_inclusive=True,
date_from=None,
date_from_inclusive=True,
date_to=None,
date_to_inclusive=True,
active_at_from=None,
active_at_from_inclusive=True,
active_at_to=None,
active_at_to_inclusive=True,
times_seen=None,
times_seen_lower=None,
times_seen_lower_inclusive=True,
times_seen_upper=None,
times_seen_upper_inclusive=True,
cursor=None,
limit=None
):
from sentry.models import Event, Group, GroupSubscription, GroupStatus
engine = get_db_engine('default')
queryset = Group.objects.filter(project=project)
if query:
# TODO(dcramer): if we want to continue to support search on SQL
# we should at least optimize this in Postgres so that it does
# the query filter **after** the index filters, and restricts the
# result set
queryset = queryset.filter(
Q(message__icontains=query) | Q(culprit__icontains=query))
if status is None:
status_in = (
GroupStatus.PENDING_DELETION, GroupStatus.DELETION_IN_PROGRESS,
GroupStatus.PENDING_MERGE,
)
queryset = queryset.exclude(status__in=status_in)
else:
queryset = queryset.filter(status=status)
if bookmarked_by:
queryset = queryset.filter(
bookmark_set__project=project,
bookmark_set__user=bookmarked_by,
)
if assigned_to:
queryset = queryset.filter(
assignee_set__project=project,
assignee_set__user=assigned_to,
)
elif unassigned in (True, False):
queryset = queryset.filter(
assignee_set__isnull=unassigned,
)
if subscribed_by is not None:
queryset = queryset.filter(
id__in=GroupSubscription.objects.filter(
project=project,
user=subscribed_by,
is_active=True,
).values_list('group'),
)
if first_release:
if first_release is EMPTY:
return queryset.none()
queryset = queryset.filter(
first_release__organization_id=project.organization_id,
first_release__version=first_release,
)
if tags:
matches = tagstore.get_tags_for_search_filter(project.id, tags)
if not matches:
return queryset.none()
queryset = queryset.filter(
id__in=matches,
)
if age_from or age_to:
params = {}
if age_from:
if age_from_inclusive:
params['first_seen__gte'] = age_from
else:
params['first_seen__gt'] = age_from
if age_to:
if age_to_inclusive:
params['first_seen__lte'] = age_to
else:
params['first_seen__lt'] = age_to
queryset = queryset.filter(**params)
if last_seen_from or last_seen_to:
params = {}
if last_seen_from:
if last_seen_from_inclusive:
params['last_seen__gte'] = last_seen_from
else:
params['last_seen__gt'] = last_seen_from
if last_seen_to:
if last_seen_to_inclusive:
params['last_seen__lte'] = last_seen_to
else:
params['last_seen__lt'] = last_seen_to
queryset = queryset.filter(**params)
if active_at_from or active_at_to:
params = {}
if active_at_from:
if active_at_from_inclusive:
params['active_at__gte'] = active_at_from
else:
params['active_at__gt'] = active_at_from
if active_at_to:
if active_at_to_inclusive:
params['active_at__lte'] = active_at_to
else:
params['active_at__lt'] = active_at_to
queryset = queryset.filter(**params)
if times_seen is not None:
queryset = queryset.filter(times_seen=times_seen)
if times_seen_lower is not None or times_seen_upper is not None:
params = {}
if times_seen_lower is not None:
if times_seen_lower_inclusive:
params['times_seen__gte'] = times_seen_lower
else:
params['times_seen__gt'] = times_seen_lower
if times_seen_upper is not None:
if times_seen_upper_inclusive:
params['times_seen__lte'] = times_seen_upper
else:
params['times_seen__lt'] = times_seen_upper
queryset = queryset.filter(**params)
if date_from or date_to:
params = {
'project_id': project.id,
}
if date_from:
if date_from_inclusive:
params['datetime__gte'] = date_from
else:
params['datetime__gt'] = date_from
if date_to:
if date_to_inclusive:
params['datetime__lte'] = date_to
else:
params['datetime__lt'] = date_to
event_queryset = Event.objects.filter(**params)
if query:
event_queryset = event_queryset.filter(
message__icontains=query)
# limit to the first 1000 results
group_ids = event_queryset.distinct().values_list(
'group_id', flat=True)[:1000]
# if Event is not on the primary database remove Django's
# implicit subquery by coercing to a list
base = router.db_for_read(Group)
using = router.db_for_read(Event)
# MySQL also cannot do a LIMIT inside of a subquery
if base != using or engine.startswith('mysql'):
group_ids = list(group_ids)
queryset = queryset.filter(
id__in=group_ids,
)
if engine.startswith('sqlite'):
score_clause = SQLITE_SORT_CLAUSES[sort_by]
elif engine.startswith('mysql'):
score_clause = MYSQL_SORT_CLAUSES[sort_by]
elif engine.startswith('oracle'):
score_clause = ORACLE_SORT_CLAUSES[sort_by]
elif engine in MSSQL_ENGINES:
score_clause = MSSQL_SORT_CLAUSES[sort_by]
else:
score_clause = SORT_CLAUSES[sort_by]
queryset = queryset.extra(
select={'sort_value': score_clause},
)
return queryset
def get_group_tags(request, organization, project, group_id, tag_name):
# XXX(dcramer): Consider this API deprecated as soon as it was implemented
cutoff = timezone.now() - timedelta(days=7)
engine = get_db_engine('default')
if 'postgres' in engine:
# This doesnt guarantee percentage is accurate, but it does ensure
# that the query has a maximum cost
cursor = connections['default'].cursor()
cursor.execute("""
SELECT SUM(t)
FROM (
SELECT times_seen as t
FROM sentry_messagefiltervalue
WHERE group_id = %s
AND key = %s
AND last_seen > NOW() - INTERVAL '7 days'
LIMIT 10000
) as a
""", [group_id, tag_name])
total = cursor.fetchone()[0] or 0
else:
total = GroupTagValue.objects.filter(
group=group_id,
key=tag_name,
last_seen__gte=cutoff,
).aggregate(t=Sum('times_seen'))['t'] or 0
unique_tags = GroupTagValue.objects.filter(
group=group_id,
key=tag_name,
last_seen__gte=cutoff,
).values_list('value', 'times_seen').order_by('-times_seen')[:10]
# fetch TagValue instances so we can get proper labels
tag_values = dict(
(t.value, t)
for t in TagValue.objects.filter(
key=tag_name,
project_id=project.id,
value__in=[u[0] for u in unique_tags],
)
)
values = []
for tag, times_seen in unique_tags:
try:
tag_value = tag_values[tag]
except KeyError:
label = tag
else:
label = tag_value.get_label()
values.append({
'value': tag,
'count': times_seen,
'label': label,
})
return json.dumps({
'name': tag_name,
'values': values,
'total': total,
})
def query(self, project, query=None, status=None, tags=None,
bookmarked_by=None, assigned_to=None, first_release=None,
sort_by='date', date_filter='last_seen', date_from=None,
date_to=None, cursor=None, limit=100):
from sentry.models import Group, GroupStatus
queryset = Group.objects.filter(project=project)
if query:
# TODO(dcramer): if we want to continue to support search on SQL
# we should at least optimize this in Postgres so that it does
# the query filter **after** the index filters, and restricts the
# result set
queryset = queryset.filter(
Q(message__icontains=query) |
Q(culprit__icontains=query)
)
if status is None:
queryset = queryset.exclude(
status__in=(
GroupStatus.PENDING_DELETION,
GroupStatus.DELETION_IN_PROGRESS,
GroupStatus.PENDING_MERGE,
)
)
else:
queryset = queryset.filter(status=status)
if bookmarked_by:
queryset = queryset.filter(
bookmark_set__project=project,
bookmark_set__user=bookmarked_by,
)
if assigned_to:
queryset = queryset.filter(
assignee_set__project=project,
assignee_set__user=assigned_to,
)
if first_release:
queryset = queryset.filter(
first_release__project=project,
first_release__version=first_release,
)
if tags:
for k, v in tags.iteritems():
queryset = queryset.filter(**dict(
grouptag__key=k,
grouptag__value=v,
))
if date_filter == 'first_seen':
if date_from and date_to:
queryset = queryset.filter(
first_seen__gte=date_from,
first_seen__lte=date_to,
)
elif date_from:
queryset = queryset.filter(first_seen__gte=date_from)
elif date_to:
queryset = queryset.filter(first_seen__lte=date_to)
elif date_filter == 'last_seen':
if date_from and date_to:
queryset = queryset.filter(
first_seen__gte=date_from,
last_seen__lte=date_to,
)
elif date_from:
queryset = queryset.filter(last_seen__gte=date_from)
elif date_to:
queryset = queryset.filter(last_seen__lte=date_to)
engine = get_db_engine('default')
if engine.startswith('sqlite'):
score_clause = SQLITE_SORT_CLAUSES[sort_by]
elif engine.startswith('mysql'):
score_clause = MYSQL_SORT_CLAUSES[sort_by]
elif engine.startswith('oracle'):
score_clause = ORACLE_SORT_CLAUSES[sort_by]
elif engine in MSSQL_ENGINES:
score_clause = MSSQL_SORT_CLAUSES[sort_by]
else:
score_clause = SORT_CLAUSES[sort_by]
if sort_by == 'tottime':
queryset = queryset.filter(time_spent_count__gt=0)
elif sort_by == 'avgtime':
queryset = queryset.filter(time_spent_count__gt=0)
queryset = queryset.extra(
select={'sort_value': score_clause},
)
# HACK: don't sort by the same column twice
if sort_by == 'date':
paginator_cls = DateTimePaginator
sort_clause = '-last_seen'
elif sort_by == 'priority':
paginator_cls = Paginator
sort_clause = '-score'
elif sort_by == 'new':
paginator_cls = DateTimePaginator
sort_clause = '-first_seen'
elif sort_by == 'freq':
paginator_cls = Paginator
sort_clause = '-times_seen'
else:
paginator_cls = Paginator
sort_clause = '-sort_value'
queryset = queryset.order_by(sort_clause)
paginator = paginator_cls(queryset, sort_clause)
return paginator.get_result(limit, cursor)
def query(self, project, query=None, status=None, tags=None,
bookmarked_by=None, sort_by='date', date_filter='last_seen',
date_from=None, date_to=None, cursor=None, limit=100):
from sentry.models import Group
queryset = Group.objects.filter(project=project)
if query:
queryset = queryset.filter(message__icontains=query)
if status is not None:
queryset = queryset.filter(status=status)
if bookmarked_by:
queryset = queryset.filter(
bookmark_set__project=project,
bookmark_set__user=bookmarked_by,
)
if tags:
for k, v in tags.iteritems():
queryset = queryset.filter(**dict(
grouptag__key=k,
grouptag__value=v,
))
if date_filter == 'first_seen':
if date_from:
queryset = queryset.filter(first_seen__gte=date_from)
elif date_to:
queryset = queryset.filter(first_seen__lte=date_to)
elif date_filter == 'last_seen':
if date_from and date_to:
queryset = queryset.filter(
first_seen__gte=date_from,
last_seen__lte=date_to,
)
elif date_from:
queryset = queryset.filter(last_seen__gte=date_from)
elif date_to:
queryset = queryset.filter(last_seen__lte=date_to)
engine = get_db_engine('default')
if engine.startswith('sqlite'):
score_clause = SQLITE_SORT_CLAUSES.get(sort_by)
filter_clause = SQLITE_SCORE_CLAUSES.get(sort_by)
elif engine.startswith('mysql'):
score_clause = MYSQL_SORT_CLAUSES.get(sort_by)
filter_clause = MYSQL_SCORE_CLAUSES.get(sort_by)
elif engine.startswith('oracle'):
score_clause = ORACLE_SORT_CLAUSES.get(sort_by)
filter_clause = ORACLE_SCORE_CLAUSES.get(sort_by)
elif engine in MSSQL_ENGINES:
score_clause = MSSQL_SORT_CLAUSES.get(sort_by)
filter_clause = MSSQL_SCORE_CLAUSES.get(sort_by)
else:
score_clause = SORT_CLAUSES.get(sort_by)
filter_clause = SCORE_CLAUSES.get(sort_by)
if sort_by == 'tottime':
queryset = queryset.filter(time_spent_count__gt=0)
elif sort_by == 'avgtime':
queryset = queryset.filter(time_spent_count__gt=0)
if score_clause:
queryset = queryset.extra(
select={'sort_value': score_clause},
)
# HACK: don't sort by the same column twice
if sort_by == 'date':
queryset = queryset.order_by('-last_seen')
else:
queryset = queryset.order_by('-sort_value', '-last_seen')
if cursor:
queryset = queryset.extra(
where=['%s > %%s' % filter_clause],
params=[float(cursor)],
)
# HACK:
return SearchResult(instances=list(queryset[:limit]))
def query(self,
project,
query=None,
status=None,
tags=None,
bookmarked_by=None,
sort_by='date',
date_filter='last_seen',
date_from=None,
date_to=None,
cursor=None,
limit=100):
from sentry.models import Group
queryset = Group.objects.filter(project=project)
if query:
queryset = queryset.filter(message__icontains=query)
if status is not None:
queryset = queryset.filter(status=status)
if bookmarked_by:
queryset = queryset.filter(
bookmark_set__project=project,
bookmark_set__user=bookmarked_by,
)
if tags:
for k, v in tags.iteritems():
queryset = queryset.filter(**dict(
grouptag__key=k,
grouptag__value=v,
))
if date_filter == 'first_seen':
if date_from:
queryset = queryset.filter(first_seen__gte=date_from)
elif date_to:
queryset = queryset.filter(first_seen__lte=date_to)
elif date_filter == 'last_seen':
if date_from and date_to:
queryset = queryset.filter(
first_seen__gte=date_from,
last_seen__lte=date_to,
)
elif date_from:
queryset = queryset.filter(last_seen__gte=date_from)
elif date_to:
queryset = queryset.filter(last_seen__lte=date_to)
engine = get_db_engine('default')
if engine.startswith('sqlite'):
score_clause = SQLITE_SORT_CLAUSES.get(sort_by)
filter_clause = SQLITE_SCORE_CLAUSES.get(sort_by)
elif engine.startswith('mysql'):
score_clause = MYSQL_SORT_CLAUSES.get(sort_by)
filter_clause = MYSQL_SCORE_CLAUSES.get(sort_by)
elif engine.startswith('oracle'):
score_clause = ORACLE_SORT_CLAUSES.get(sort_by)
filter_clause = ORACLE_SCORE_CLAUSES.get(sort_by)
elif engine in MSSQL_ENGINES:
score_clause = MSSQL_SORT_CLAUSES.get(sort_by)
filter_clause = MSSQL_SCORE_CLAUSES.get(sort_by)
else:
score_clause = SORT_CLAUSES.get(sort_by)
filter_clause = SCORE_CLAUSES.get(sort_by)
if sort_by == 'tottime':
queryset = queryset.filter(time_spent_count__gt=0)
elif sort_by == 'avgtime':
queryset = queryset.filter(time_spent_count__gt=0)
if score_clause:
queryset = queryset.extra(select={'sort_value': score_clause}, )
# HACK: don't sort by the same column twice
if sort_by == 'date':
queryset = queryset.order_by('-last_seen')
else:
queryset = queryset.order_by('-sort_value', '-last_seen')
if cursor:
queryset = queryset.extra(
where=['%s > %%s' % filter_clause],
params=[float(cursor)],
)
# HACK:
return SearchResult(instances=queryset[:limit])
def query(
self,
project,
query=None,
status=None,
tags=None,
bookmarked_by=None,
assigned_to=None,
first_release=None,
sort_by="date",
age_from=None,
age_to=None,
unassigned=None,
date_from=None,
date_to=None,
cursor=None,
limit=100,
):
from sentry.models import Event, Group, GroupStatus
queryset = Group.objects.filter(project=project)
if query:
# TODO(dcramer): if we want to continue to support search on SQL
# we should at least optimize this in Postgres so that it does
# the query filter **after** the index filters, and restricts the
# result set
queryset = queryset.filter(Q(message__icontains=query) | Q(culprit__icontains=query))
if status is None:
queryset = queryset.exclude(
status__in=(GroupStatus.PENDING_DELETION, GroupStatus.DELETION_IN_PROGRESS, GroupStatus.PENDING_MERGE)
)
else:
queryset = queryset.filter(status=status)
if bookmarked_by:
queryset = queryset.filter(bookmark_set__project=project, bookmark_set__user=bookmarked_by)
if assigned_to:
queryset = queryset.filter(assignee_set__project=project, assignee_set__user=assigned_to)
elif unassigned in (True, False):
queryset = queryset.filter(assignee_set__isnull=unassigned)
if first_release:
queryset = queryset.filter(first_release__project=project, first_release__version=first_release)
if tags:
for k, v in tags.iteritems():
if v == ANY:
queryset = queryset.filter(grouptag__key=k)
else:
queryset = queryset.filter(grouptag__key=k, grouptag__value=v)
if age_from and age_to:
queryset = queryset.filter(first_seen__gte=age_from, first_seen__lte=age_to)
elif age_from:
queryset = queryset.filter(first_seen__gte=age_from)
elif age_to:
queryset = queryset.filter(first_seen__lte=age_to)
if date_from or date_to:
if date_from and date_to:
event_queryset = Event.objects.filter(
project_id=project.id, datetime__gte=date_from, datetime__lte=date_to
)
elif date_from:
event_queryset = Event.objects.filter(project_id=project.id, datetime__gte=date_from)
elif date_to:
event_queryset = Event.objects.filter(project_id=project.id, datetime__lte=date_to)
# limit to the first 1000 results
group_ids = event_queryset.distinct().values_list("group_id", flat=True)[:1000]
queryset = queryset.filter(id__in=group_ids)
engine = get_db_engine("default")
if engine.startswith("sqlite"):
score_clause = SQLITE_SORT_CLAUSES[sort_by]
elif engine.startswith("mysql"):
score_clause = MYSQL_SORT_CLAUSES[sort_by]
elif engine.startswith("oracle"):
score_clause = ORACLE_SORT_CLAUSES[sort_by]
elif engine in MSSQL_ENGINES:
score_clause = MSSQL_SORT_CLAUSES[sort_by]
else:
score_clause = SORT_CLAUSES[sort_by]
if sort_by == "tottime":
queryset = queryset.filter(time_spent_count__gt=0)
elif sort_by == "avgtime":
queryset = queryset.filter(time_spent_count__gt=0)
queryset = queryset.extra(select={"sort_value": score_clause})
# HACK: don't sort by the same column twice
if sort_by == "date":
paginator_cls = DateTimePaginator
sort_clause = "-last_seen"
elif sort_by == "priority":
paginator_cls = Paginator
sort_clause = "-score"
elif sort_by == "new":
paginator_cls = DateTimePaginator
sort_clause = "-first_seen"
elif sort_by == "freq":
paginator_cls = Paginator
sort_clause = "-times_seen"
else:
paginator_cls = Paginator
sort_clause = "-sort_value"
queryset = queryset.order_by(sort_clause)
paginator = paginator_cls(queryset, sort_clause)
return paginator.get_result(limit, cursor)
def get_accelerated(self, queryset=None, minutes=15):
# mintues should
from sentry.models import MessageCountByMinute
mcbm_tbl = MessageCountByMinute._meta.db_table
if queryset is None:
queryset = self
normalization = float(settings.MINUTE_NORMALIZATION)
assert minutes >= normalization
engine = get_db_engine(queryset.db)
# We technically only support mysql and postgresql, since there seems to be no standard
# way to get the epoch from a datetime/interval
if engine.startswith('mysql'):
minute_clause = "interval %s minute"
epoch_clause = "unix_timestamp(utc_timestamp()) - unix_timestamp(%(mcbm_tbl)s.date)"
now_clause = 'utc_timestamp()'
else:
minute_clause = "interval '%s minutes'"
epoch_clause = "extract(epoch from now()) - extract(epoch from %(mcbm_tbl)s.date)"
now_clause = 'now()'
epoch_clause = epoch_clause % dict(mcbm_tbl=mcbm_tbl)
queryset = queryset.extra(
where=[
"%s.date >= %s - %s" % (mcbm_tbl, now_clause, minute_clause % (minutes + 1, )),
"%s.date <= %s - %s" % (mcbm_tbl, now_clause, minute_clause % (1, ))
],
).annotate(x=Sum('messagecountbyminute__times_seen')).order_by('id')
sql, params = queryset.query.get_compiler(queryset.db).as_sql()
before_select, after_select = str(sql).split('SELECT ', 1)
before_where, after_where = after_select.split(' WHERE ', 1)
before_group, after_group = after_where.split(' GROUP BY ', 1)
# Ensure we remove any ordering clause
after_group = after_group.split(' ORDER BY ')[0]
query = """
SELECT DISTINCT (SUM(%(mcbm_tbl)s.times_seen) * (%(norm)f / (%(epoch_clause)s / 60)) + 1.0) / (COALESCE(z.rate, 0) + 1.0) as accel,
(COALESCE(z.rate, 0) + 1.0) as prev_rate,
%(before_where)s
LEFT JOIN (SELECT a.group_id, SUM(a.times_seen) / COUNT(a.times_seen) / %(norm)f as rate
FROM %(mcbm_tbl)s as a
WHERE a.date BETWEEN %(now)s - %(max_time)s
AND %(now)s - %(min_time)s
GROUP BY a.group_id) as z
ON z.group_id = %(mcbm_tbl)s.group_id
WHERE %(mcbm_tbl)s.date BETWEEN %(now)s - %(min_time)s
AND %(now)s - %(offset_time)s
AND %(before_group)s
GROUP BY prev_rate, %(mcbm_tbl)s.date, %(after_group)s
HAVING SUM(%(mcbm_tbl)s.times_seen) > 0
ORDER BY accel DESC
""" % dict(
mcbm_tbl=mcbm_tbl,
before_where=before_where,
before_group=before_group,
after_group=after_group,
offset_time=minute_clause % (1,),
min_time=minute_clause % (minutes + 1,),
max_time=minute_clause % (minutes * (60 / normalization),),
norm=normalization,
epoch_clause=epoch_clause,
now=now_clause,
)
return RawQuerySet(self, query, params)
def test_with_dotted_path(self):
with self.Settings(DATABASES={'default': {'ENGINE': 'blah.sqlite3'}}):
self.assertEquals(get_db_engine(), 'sqlite3')
def test_no_path(self):
with self.Settings(DATABASES={'default': {'ENGINE': 'mysql'}}):
self.assertEquals(get_db_engine(), 'mysql')
def test_with_dotted_path(self):
with self.settings(DATABASES={'default': {'ENGINE': 'blah.postgres'}}):
self.assertEquals(get_db_engine(), 'postgres')
def query(self, project, query=None, status=None, tags=None,
bookmarked_by=None, assigned_to=None, first_release=None,
sort_by='date', unassigned=None,
age_from=None, age_from_inclusive=True,
age_to=None, age_to_inclusive=True,
date_from=None, date_from_inclusive=True,
date_to=None, date_to_inclusive=True,
cursor=None, limit=100):
from sentry.models import Event, Group, GroupStatus
queryset = Group.objects.filter(project=project)
if query:
# TODO(dcramer): if we want to continue to support search on SQL
# we should at least optimize this in Postgres so that it does
# the query filter **after** the index filters, and restricts the
# result set
queryset = queryset.filter(
Q(message__icontains=query) |
Q(culprit__icontains=query)
)
if status is None:
queryset = queryset.exclude(
status__in=(
GroupStatus.PENDING_DELETION,
GroupStatus.DELETION_IN_PROGRESS,
GroupStatus.PENDING_MERGE,
)
)
else:
queryset = queryset.filter(status=status)
if bookmarked_by:
queryset = queryset.filter(
bookmark_set__project=project,
bookmark_set__user=bookmarked_by,
)
if assigned_to:
queryset = queryset.filter(
assignee_set__project=project,
assignee_set__user=assigned_to,
)
elif unassigned in (True, False):
queryset = queryset.filter(
assignee_set__isnull=unassigned,
)
if first_release:
queryset = queryset.filter(
first_release__project=project,
first_release__version=first_release,
)
if tags:
for k, v in tags.iteritems():
if v == ANY:
queryset = queryset.filter(
grouptag__project=project,
grouptag__key=k,
)
else:
queryset = queryset.filter(
grouptag__project=project,
grouptag__key=k,
grouptag__value=v,
)
queryset = queryset.distinct()
if age_from or age_to:
params = {}
if age_from:
if age_from_inclusive:
params['first_seen__gte'] = age_from
else:
params['first_seen__gt'] = age_from
if age_to:
if age_to_inclusive:
params['first_seen__lte'] = age_to
else:
params['first_seen__lt'] = age_to
queryset = queryset.filter(**params)
if date_from or date_to:
params = {
'project_id': project.id,
}
if date_from:
if date_from_inclusive:
params['datetime__gte'] = date_from
else:
params['datetime__gt'] = date_from
if date_to:
if date_to_inclusive:
params['datetime__lte'] = date_to
else:
params['datetime__lt'] = date_to
event_queryset = Event.objects.filter(**params)
# limit to the first 1000 results
group_ids = event_queryset.distinct().values_list(
'group_id',
flat=True
)[:1000]
# if Event is not on the primary database remove Django's
# implicit subquery by coercing to a list
base = router.db_for_read(Group)
using = router.db_for_read(Event)
if base != using:
group_ids = list(group_ids)
queryset = queryset.filter(
id__in=group_ids,
)
engine = get_db_engine('default')
if engine.startswith('sqlite'):
score_clause = SQLITE_SORT_CLAUSES[sort_by]
elif engine.startswith('mysql'):
score_clause = MYSQL_SORT_CLAUSES[sort_by]
elif engine.startswith('oracle'):
score_clause = ORACLE_SORT_CLAUSES[sort_by]
elif engine in MSSQL_ENGINES:
score_clause = MSSQL_SORT_CLAUSES[sort_by]
else:
score_clause = SORT_CLAUSES[sort_by]
if sort_by == 'tottime':
queryset = queryset.filter(time_spent_count__gt=0)
elif sort_by == 'avgtime':
queryset = queryset.filter(time_spent_count__gt=0)
queryset = queryset.extra(
select={'sort_value': score_clause},
)
# HACK: don't sort by the same column twice
if sort_by == 'date':
paginator_cls = DateTimePaginator
sort_clause = '-last_seen'
elif sort_by == 'priority':
paginator_cls = Paginator
sort_clause = '-score'
elif sort_by == 'new':
paginator_cls = DateTimePaginator
sort_clause = '-first_seen'
elif sort_by == 'freq':
paginator_cls = Paginator
sort_clause = '-times_seen'
else:
paginator_cls = Paginator
sort_clause = '-sort_value'
queryset = queryset.order_by(sort_clause)
paginator = paginator_cls(queryset, sort_clause)
return paginator.get_result(limit, cursor)
def _build_queryset(self,
project,
query=None,
status=None,
tags=None,
bookmarked_by=None,
assigned_to=None,
first_release=None,
sort_by='date',
unassigned=None,
age_from=None,
age_from_inclusive=True,
age_to=None,
age_to_inclusive=True,
date_from=None,
date_from_inclusive=True,
date_to=None,
date_to_inclusive=True,
cursor=None,
limit=None):
from sentry.models import Event, Group, GroupStatus
engine = get_db_engine('default')
queryset = Group.objects.filter(project=project)
if query:
# TODO(dcramer): if we want to continue to support search on SQL
# we should at least optimize this in Postgres so that it does
# the query filter **after** the index filters, and restricts the
# result set
queryset = queryset.filter(
Q(message__icontains=query) | Q(culprit__icontains=query))
if status is None:
queryset = queryset.exclude(status__in=(
GroupStatus.PENDING_DELETION,
GroupStatus.DELETION_IN_PROGRESS,
GroupStatus.PENDING_MERGE,
))
else:
queryset = queryset.filter(status=status)
if bookmarked_by:
queryset = queryset.filter(
bookmark_set__project=project,
bookmark_set__user=bookmarked_by,
)
if assigned_to:
queryset = queryset.filter(
assignee_set__project=project,
assignee_set__user=assigned_to,
)
elif unassigned in (True, False):
queryset = queryset.filter(assignee_set__isnull=unassigned, )
if first_release:
if first_release is EMPTY:
return queryset.none()
queryset = queryset.filter(
first_release__project=project,
first_release__version=first_release,
)
if tags:
matches = self._tags_to_filter(project, tags)
if not matches:
return queryset.none()
queryset = queryset.filter(id__in=matches, )
if age_from or age_to:
params = {}
if age_from:
if age_from_inclusive:
params['first_seen__gte'] = age_from
else:
params['first_seen__gt'] = age_from
if age_to:
if age_to_inclusive:
params['first_seen__lte'] = age_to
else:
params['first_seen__lt'] = age_to
queryset = queryset.filter(**params)
if date_from or date_to:
params = {
'project_id': project.id,
}
if date_from:
if date_from_inclusive:
params['datetime__gte'] = date_from
else:
params['datetime__gt'] = date_from
if date_to:
if date_to_inclusive:
params['datetime__lte'] = date_to
else:
params['datetime__lt'] = date_to
event_queryset = Event.objects.filter(**params)
# limit to the first 1000 results
group_ids = event_queryset.distinct().values_list('group_id',
flat=True)[:1000]
# if Event is not on the primary database remove Django's
# implicit subquery by coercing to a list
base = router.db_for_read(Group)
using = router.db_for_read(Event)
# MySQL also cannot do a LIMIT inside of a subquery
if base != using or engine.startswith('mysql'):
group_ids = list(group_ids)
queryset = queryset.filter(id__in=group_ids, )
if engine.startswith('sqlite'):
score_clause = SQLITE_SORT_CLAUSES[sort_by]
elif engine.startswith('mysql'):
score_clause = MYSQL_SORT_CLAUSES[sort_by]
elif engine.startswith('oracle'):
score_clause = ORACLE_SORT_CLAUSES[sort_by]
elif engine in MSSQL_ENGINES:
score_clause = MSSQL_SORT_CLAUSES[sort_by]
else:
score_clause = SORT_CLAUSES[sort_by]
queryset = queryset.extra(select={'sort_value': score_clause}, )
return queryset
def get_accelerated(self, project_ids, queryset=None, minutes=15):
if not project_ids:
return self.none()
if queryset is None:
queryset = self.filter(
project__in=project_ids,
status=STATUS_UNRESOLVED,
)
else:
queryset = queryset._clone()
queryset.query.select_related = False
normalization = float(MINUTE_NORMALIZATION)
assert minutes >= normalization
intervals = 8
engine = get_db_engine(queryset.db)
# We technically only support mysql and postgresql, since there seems to be no standard
# way to get the epoch from a datetime/interval
if engine.startswith('mysql'):
minute_clause = "interval %s minute"
epoch_clause = "unix_timestamp(utc_timestamp()) - unix_timestamp(mcbm.date)"
now_clause = 'utc_timestamp()'
else:
minute_clause = "interval '%s minutes'"
epoch_clause = "extract(epoch from now()) - extract(epoch from mcbm.date)"
now_clause = 'now()'
sql, params = queryset.query.get_compiler(queryset.db).as_sql()
before_select, after_select = str(sql).split('SELECT ', 1)
after_where = after_select.split(' WHERE ', 1)[1]
# Ensure we remove any ordering clause
after_where = after_where.split(' ORDER BY ')[0]
query = """
SELECT ((mcbm.times_seen + 1) / ((%(epoch_clause)s) / 60)) / (COALESCE(z.rate, 0) + 1) as sort_value,
%(fields)s
FROM sentry_groupedmessage
INNER JOIN sentry_messagecountbyminute as mcbm
ON (sentry_groupedmessage.id = mcbm.group_id)
LEFT JOIN (SELECT a.group_id, (SUM(a.times_seen)) / COUNT(a.times_seen) / %(norm)f as rate
FROM sentry_messagecountbyminute as a
WHERE a.date >= %(now)s - %(max_time)s
AND a.date < %(now)s - %(min_time)s
AND a.project_id IN (%(project_ids)s)
GROUP BY a.group_id) as z
ON z.group_id = mcbm.group_id
WHERE mcbm.date >= %(now)s - %(min_time)s
AND mcbm.date < %(now)s - %(offset_time)s
AND mcbm.times_seen > 0
AND ((mcbm.times_seen + 1) / ((%(epoch_clause)s) / 60)) > (COALESCE(z.rate, 0) + 1)
AND %(after_where)s
GROUP BY z.rate, mcbm.times_seen, mcbm.date, %(fields)s
ORDER BY sort_value DESC
""" % dict(
fields=self.model_fields_clause,
after_where=after_where,
offset_time=minute_clause % (1,),
min_time=minute_clause % (minutes + 1,),
max_time=minute_clause % (minutes * intervals + 1,),
norm=normalization,
epoch_clause=epoch_clause,
now=now_clause,
project_ids=', '.join((str(int(x)) for x in project_ids)),
)
return RawQuerySet(self, query, params)
messagecountbyminute__date__gte=date_from)
if date_to:
if not date_from:
event_list = event_list.filter(last_seen__lte=date_to)
else:
event_list = event_list.filter(
messagecountbyminute__date__lte=date_to)
sort = request.GET.get('sort')
if sort not in SORT_OPTIONS:
sort = settings.DEFAULT_SORT_OPTION
if sort.startswith('accel_') and not has_trending():
sort = settings.DEFAULT_SORT_OPTION
engine = get_db_engine('default')
if engine.startswith('sqlite'):
sort_clause = SQLITE_SORT_CLAUSES.get(sort)
elif engine.startswith('mysql'):
sort_clause = MYSQL_SORT_CLAUSES.get(sort)
else:
sort_clause = SORT_CLAUSES.get(sort)
# All filters must already be applied once we reach this point
if sort == 'tottime':
event_list = event_list.filter(time_spent_count__gt=0)
elif sort == 'avgtime':
event_list = event_list.filter(time_spent_count__gt=0)
elif sort.startswith('accel_'):
event_list = Group.objects.get_accelerated(event_list,
minutes=int(
def _get_group_list(request, project):
filters = []
for cls in get_filters(Group, project):
try:
filters.append(cls(request, project))
except Exception as e:
logger = logging.getLogger('sentry.filters')
logger.exception('Error initializing filter %r: %s', cls, e)
event_list = Group.objects
if request.user.is_authenticated() and request.GET.get('bookmarks'):
event_list = event_list.filter(
bookmark_set__project=project,
bookmark_set__user=request.user,
)
else:
event_list = event_list.filter(project=project)
for filter_ in filters:
try:
if not filter_.is_set():
continue
event_list = filter_.get_query_set(event_list)
except Exception as e:
logger = logging.getLogger('sentry.filters')
logger.exception('Error processing filter %r: %s', cls, e)
date_from = request.GET.get('df')
time_from = request.GET.get('tf')
date_to = request.GET.get('dt')
time_to = request.GET.get('tt')
date_type = request.GET.get('date_type')
today = timezone.now()
# date format is Y-m-d
if any(x is not None for x in [date_from, time_from, date_to, time_to]):
date_from, date_to = parse_date(date_from, time_from), parse_date(date_to, time_to)
else:
date_from = today - datetime.timedelta(days=5)
date_to = None
if date_type == 'first_seen':
if date_from:
event_list = event_list.filter(first_seen__gte=date_from)
elif date_to:
event_list = event_list.filter(first_seen__lte=date_to)
else:
if date_from and date_to:
event_list = event_list.filter(
first_seen__gte=date_from,
last_seen__lte=date_to,
)
elif date_from:
event_list = event_list.filter(last_seen__gte=date_from)
elif date_to:
event_list = event_list.filter(last_seen__lte=date_to)
sort = request.GET.get('sort') or request.session.get('streamsort')
if sort not in SORT_OPTIONS:
sort = DEFAULT_SORT_OPTION
# Save last sort in session
if sort != request.session.get('streamsort'):
request.session['streamsort'] = sort
engine = get_db_engine('default')
if engine.startswith('sqlite'):
score_clause = SQLITE_SORT_CLAUSES.get(sort)
filter_clause = SQLITE_SCORE_CLAUSES.get(sort)
elif engine.startswith('mysql'):
score_clause = MYSQL_SORT_CLAUSES.get(sort)
filter_clause = MYSQL_SCORE_CLAUSES.get(sort)
elif engine.startswith('oracle'):
score_clause = ORACLE_SORT_CLAUSES.get(sort)
filter_clause = ORACLE_SCORE_CLAUSES.get(sort)
elif engine in ('django_pytds', 'sqlserver_ado', 'sql_server.pyodbc'):
score_clause = MSSQL_SORT_CLAUSES.get(sort)
filter_clause = MSSQL_SCORE_CLAUSES.get(sort)
else:
score_clause = SORT_CLAUSES.get(sort)
filter_clause = SCORE_CLAUSES.get(sort)
# IMPORTANT: All filters must already be applied once we reach this point
if sort == 'tottime':
event_list = event_list.filter(time_spent_count__gt=0)
elif sort == 'avgtime':
event_list = event_list.filter(time_spent_count__gt=0)
if score_clause:
event_list = event_list.extra(
select={'sort_value': score_clause},
)
# HACK: don't sort by the same column twice
if sort == 'date':
event_list = event_list.order_by('-last_seen')
else:
event_list = event_list.order_by('-sort_value', '-last_seen')
cursor = request.GET.get('cursor', request.GET.get('c'))
if cursor:
event_list = event_list.extra(
where=['%s > %%s' % filter_clause],
params=[float(cursor)],
)
return {
'filters': filters,
'event_list': event_list,
'date_from': date_from,
'date_to': date_to,
'today': today,
'sort': sort,
'date_type': date_type
}
def _build_queryset(
self,
project,
query=None,
status=None,
tags=None,
bookmarked_by=None,
assigned_to=None,
first_release=None,
sort_by="date",
unassigned=None,
age_from=None,
age_from_inclusive=True,
age_to=None,
age_to_inclusive=True,
date_from=None,
date_from_inclusive=True,
date_to=None,
date_to_inclusive=True,
cursor=None,
limit=None,
):
from sentry.models import Event, Group, GroupStatus
engine = get_db_engine("default")
queryset = Group.objects.filter(project=project)
if query:
# TODO(dcramer): if we want to continue to support search on SQL
# we should at least optimize this in Postgres so that it does
# the query filter **after** the index filters, and restricts the
# result set
queryset = queryset.filter(Q(message__icontains=query) | Q(culprit__icontains=query))
if status is None:
queryset = queryset.exclude(
status__in=(GroupStatus.PENDING_DELETION, GroupStatus.DELETION_IN_PROGRESS, GroupStatus.PENDING_MERGE)
)
else:
queryset = queryset.filter(status=status)
if bookmarked_by:
queryset = queryset.filter(bookmark_set__project=project, bookmark_set__user=bookmarked_by)
if assigned_to:
queryset = queryset.filter(assignee_set__project=project, assignee_set__user=assigned_to)
elif unassigned in (True, False):
queryset = queryset.filter(assignee_set__isnull=unassigned)
if first_release:
if first_release is EMPTY:
return queryset.none()
queryset = queryset.filter(first_release__project=project, first_release__version=first_release)
if tags:
matches = self._tags_to_filter(project, tags)
if not matches:
return queryset.none()
queryset = queryset.filter(id__in=matches)
if age_from or age_to:
params = {}
if age_from:
if age_from_inclusive:
params["first_seen__gte"] = age_from
else:
params["first_seen__gt"] = age_from
if age_to:
if age_to_inclusive:
params["first_seen__lte"] = age_to
else:
params["first_seen__lt"] = age_to
queryset = queryset.filter(**params)
if date_from or date_to:
params = {"project_id": project.id}
if date_from:
if date_from_inclusive:
params["datetime__gte"] = date_from
else:
params["datetime__gt"] = date_from
if date_to:
if date_to_inclusive:
params["datetime__lte"] = date_to
else:
params["datetime__lt"] = date_to
event_queryset = Event.objects.filter(**params)
# limit to the first 1000 results
group_ids = event_queryset.distinct().values_list("group_id", flat=True)[:1000]
# if Event is not on the primary database remove Django's
# implicit subquery by coercing to a list
base = router.db_for_read(Group)
using = router.db_for_read(Event)
# MySQL also cannot do a LIMIT inside of a subquery
if base != using or engine.startswith("mysql"):
group_ids = list(group_ids)
queryset = queryset.filter(id__in=group_ids)
if engine.startswith("sqlite"):
score_clause = SQLITE_SORT_CLAUSES[sort_by]
elif engine.startswith("mysql"):
score_clause = MYSQL_SORT_CLAUSES[sort_by]
elif engine.startswith("oracle"):
score_clause = ORACLE_SORT_CLAUSES[sort_by]
elif engine in MSSQL_ENGINES:
score_clause = MSSQL_SORT_CLAUSES[sort_by]
else:
score_clause = SORT_CLAUSES[sort_by]
queryset = queryset.extra(select={"sort_value": score_clause})
return queryset
try:
if not filter_.is_set():
continue
event_list = filter_.get_query_set(event_list)
except Exception, e:
logger = logging.getLogger('sentry.filters')
logger.exception('Error processing filter %r: %s', cls, e)
sort = request.GET.get('sort')
if sort not in SORT_OPTIONS:
sort = settings.DEFAULT_SORT_OPTION
if sort.startswith('accel_') and not has_trending():
sort = settings.DEFAULT_SORT_OPTION
engine = get_db_engine('default')
if engine.startswith('sqlite'):
sort_clause = SQLITE_SORT_CLAUSES.get(sort)
elif engine.startswith('mysql'):
sort_clause = MYSQL_SORT_CLAUSES.get(sort)
else:
sort_clause = SORT_CLAUSES.get(sort)
if sort == 'tottime':
event_list = event_list.filter(time_spent_count__gt=0)
elif sort == 'avgtime':
event_list = event_list.filter(time_spent_count__gt=0)
elif sort.startswith('accel_'):
event_list = Group.objects.get_accelerated(event_list, minutes=int(sort.split('_', 1)[1]))
date_from = request.GET.get('df')
def get_accelerated(self, project_ids, queryset=None, minutes=15):
if not project_ids:
return self.none()
if queryset is None:
queryset = self.filter(
project__in=project_ids,
status=STATUS_UNRESOLVED,
)
else:
queryset = queryset._clone()
queryset.query.select_related = False
normalization = float(MINUTE_NORMALIZATION)
assert minutes >= normalization
intervals = 8
engine = get_db_engine(queryset.db)
# We technically only support mysql and postgresql, since there seems to be no standard
# way to get the epoch from a datetime/interval
if engine.startswith('mysql'):
minute_clause = "interval %s minute"
epoch_clause = "unix_timestamp(utc_timestamp()) - unix_timestamp(mcbm.date)"
now_clause = 'utc_timestamp()'
else:
minute_clause = "interval '%s minutes'"
epoch_clause = "extract(epoch from now()) - extract(epoch from mcbm.date)"
now_clause = 'now()'
sql, params = queryset.query.get_compiler(queryset.db).as_sql()
before_select, after_select = str(sql).split('SELECT ', 1)
after_where = after_select.split(' WHERE ', 1)[1]
# Ensure we remove any ordering clause
after_where = after_where.split(' ORDER BY ')[0]
query = """
SELECT ((mcbm.times_seen + 1) / ((%(epoch_clause)s) / 60)) / (COALESCE(z.rate, 0) + 1) as sort_value,
%(fields)s
FROM sentry_groupedmessage
INNER JOIN sentry_messagecountbyminute as mcbm
ON (sentry_groupedmessage.id = mcbm.group_id)
LEFT JOIN (SELECT a.group_id, (SUM(a.times_seen)) / COUNT(a.times_seen) / %(norm)f as rate
FROM sentry_messagecountbyminute as a
WHERE a.date >= %(now)s - %(max_time)s
AND a.date < %(now)s - %(min_time)s
AND a.project_id IN (%(project_ids)s)
GROUP BY a.group_id) as z
ON z.group_id = mcbm.group_id
WHERE mcbm.date >= %(now)s - %(min_time)s
AND mcbm.date < %(now)s - %(offset_time)s
AND mcbm.times_seen > 0
AND ((mcbm.times_seen + 1) / ((%(epoch_clause)s) / 60)) > (COALESCE(z.rate, 0) + 1)
AND %(after_where)s
GROUP BY z.rate, mcbm.times_seen, mcbm.date, %(fields)s
ORDER BY sort_value DESC
""" % dict(
fields=self.model_fields_clause,
after_where=after_where,
offset_time=minute_clause % (1, ),
min_time=minute_clause % (minutes + 1, ),
max_time=minute_clause % (minutes * intervals + 1, ),
norm=normalization,
epoch_clause=epoch_clause,
now=now_clause,
project_ids=', '.join((str(int(x)) for x in project_ids)),
)
return RawQuerySet(self, query, params)
def get(self, request, project_id):
project = Project.objects.get(
id=project_id,
)
assert_perm(project, request.user, request.auth)
group_list = Group.objects.all()
if request.user.is_authenticated() and request.GET.get('bookmarks'):
group_list = group_list.filter(
bookmark_set__project=project,
bookmark_set__user=request.user,
)
else:
group_list = group_list.filter(project=project)
status = request.GET.get('status')
if status:
group_list = group_list.filter(status=int(status))
tag_keys = TagKey.objects.all_keys(project)
for tag in tag_keys:
value = request.GET.get(tag)
if value:
group_list = group_list.filter(
grouptag__project=project,
grouptag__key=tag,
grouptag__value=value,
)
# TODO: dates should include timestamps
date_from = request.GET.get('since')
time_from = request.GET.get('until')
date_filter = request.GET.get('date_filter')
date_to = request.GET.get('dt')
time_to = request.GET.get('tt')
today = timezone.now()
# date format is Y-m-d
if any(x is not None for x in [date_from, time_from, date_to, time_to]):
date_from, date_to = parse_date(date_from, time_from), parse_date(date_to, time_to)
else:
date_from = today - timedelta(days=5)
date_to = None
if date_filter == 'first_seen':
if date_from:
group_list = group_list.filter(first_seen__gte=date_from)
elif date_to:
group_list = group_list.filter(first_seen__lte=date_to)
else:
# TODO(dcramer): a date_to no longer makes a lot of sense, and will
# need corrected when search lands
if date_from:
group_list = group_list.filter(last_seen__gte=date_from)
if date_to:
group_list = group_list.filter(last_seen__lte=date_to)
sort = request.GET.get('sort') or request.session.get('streamsort')
if sort is None:
sort = DEFAULT_SORT_OPTION
elif sort not in SORT_OPTIONS or sort.startswith('accel_'):
return HttpResponse(status=400)
# Save last sort in session
if sort != request.session.get('streamsort'):
request.session['streamsort'] = sort
engine = get_db_engine('default')
if engine.startswith('sqlite'):
score_clause = SQLITE_SORT_CLAUSES.get(sort)
filter_clause = SQLITE_SCORE_CLAUSES.get(sort)
elif engine.startswith('mysql'):
score_clause = MYSQL_SORT_CLAUSES.get(sort)
filter_clause = MYSQL_SCORE_CLAUSES.get(sort)
elif engine.startswith('oracle'):
score_clause = ORACLE_SORT_CLAUSES.get(sort)
filter_clause = ORACLE_SCORE_CLAUSES.get(sort)
elif engine in ('django_pytds', 'sqlserver_ado', 'sql_server.pyodbc'):
score_clause = MSSQL_SORT_CLAUSES.get(sort)
filter_clause = MSSQL_SCORE_CLAUSES.get(sort)
else:
score_clause = SORT_CLAUSES.get(sort)
filter_clause = SCORE_CLAUSES.get(sort)
assert score_clause
if sort == 'tottime':
group_list = group_list.filter(time_spent_count__gt=0)
elif sort == 'avgtime':
group_list = group_list.filter(time_spent_count__gt=0)
group_list = group_list.extra(
select={'sort_value': score_clause},
)
return self.paginate(
request=request,
queryset=group_list,
order_by='-sort_value',
on_results=lambda x: serialize(x, request.user),
)
def get_group_tags(request, organization, project, group_id, tag_name):
# XXX(dcramer): Consider this API deprecated as soon as it was implemented
cutoff = timezone.now() - timedelta(days=7)
engine = get_db_engine('default')
if 'postgres' in engine:
# This doesnt guarantee percentage is accurate, but it does ensure
# that the query has a maximum cost
cursor = connections['default'].cursor()
cursor.execute("""
SELECT SUM(t)
FROM (
SELECT times_seen as t
FROM sentry_messagefiltervalue
WHERE group_id = %s
AND key = %s
AND last_seen > NOW() - INTERVAL '7 days'
LIMIT 10000
) as a
""", [group_id, tag_name])
total = cursor.fetchone()[0] or 0
else:
total = GroupTagValue.objects.filter(
group=group_id,
key=tag_name,
last_seen__gte=cutoff,
).aggregate(t=Sum('times_seen'))['t'] or 0
unique_tags = GroupTagValue.objects.filter(
group=group_id,
key=tag_name,
last_seen__gte=cutoff,
).values_list('value', 'times_seen').order_by('-times_seen')[:10]
# fetch TagValue instances so we can get proper labels
tag_values = dict(
(t.value, t)
for t in TagValue.objects.filter(
key=tag_name,
project_id=project.id,
value__in=[u[0] for u in unique_tags],
)
)
values = []
for tag, times_seen in unique_tags:
try:
tag_value = tag_values[tag]
except KeyError:
label = tag
else:
label = tag_value.get_label()
values.append({
'value': tag,
'count': times_seen,
'label': label,
})
return json.dumps({
'name': tag_name,
'values': values,
'total': total,
})
def get_accelerated(self, project_ids, queryset=None, minutes=15):
# mintues should
from sentry.models import MessageCountByMinute
mcbm_tbl = MessageCountByMinute._meta.db_table
if queryset is None:
queryset = self
normalization = float(settings.MINUTE_NORMALIZATION)
assert minutes >= normalization
engine = get_db_engine(queryset.db)
# We technically only support mysql and postgresql, since there seems to be no standard
# way to get the epoch from a datetime/interval
if engine.startswith("mysql"):
minute_clause = "interval %s minute"
epoch_clause = "unix_timestamp(utc_timestamp()) - unix_timestamp(%(mcbm_tbl)s.date)"
now_clause = "utc_timestamp()"
else:
minute_clause = "interval '%s minutes'"
epoch_clause = "extract(epoch from now()) - extract(epoch from %(mcbm_tbl)s.date)"
now_clause = "now()"
epoch_clause = epoch_clause % dict(mcbm_tbl=mcbm_tbl)
queryset = queryset.annotate(x=Sum("messagecountbyminute__times_seen")).order_by("id")
sql, params = queryset.query.get_compiler(queryset.db).as_sql()
before_select, after_select = str(sql).split("SELECT ", 1)
before_where, after_where = after_select.split(" WHERE ", 1)
before_group, after_group = after_where.split(" GROUP BY ", 1)
# Ensure we remove any ordering clause
after_group = after_group.split(" ORDER BY ")[0]
# TODO: adding project_id to sort clause on left join helps query in many cases
query = """
SELECT (SUM(%(mcbm_tbl)s.times_seen) * (%(norm)f / (%(epoch_clause)s / 60)) + 1.0) / (COALESCE(z.rate, 0) + 1.0) as accel,
(COALESCE(z.rate, 0) + 1.0) as prev_rate,
%(before_where)s
LEFT JOIN (SELECT a.group_id, SUM(a.times_seen) / COUNT(a.times_seen) / %(norm)f as rate
FROM %(mcbm_tbl)s as a
WHERE a.date >= %(now)s - %(max_time)s
AND a.date < %(now)s - %(min_time)s
AND a.project_id IN (%(project_ids)s)
GROUP BY a.group_id) as z
ON z.group_id = %(mcbm_tbl)s.group_id
WHERE %(mcbm_tbl)s.date >= %(now)s - %(min_time)s
AND %(mcbm_tbl)s.date < %(now)s - %(offset_time)s
AND %(before_group)s
GROUP BY prev_rate, %(mcbm_tbl)s.date, %(after_group)s
HAVING SUM(%(mcbm_tbl)s.times_seen) > 0
ORDER BY accel DESC
""" % dict(
mcbm_tbl=mcbm_tbl,
before_where=before_where,
before_group=before_group,
after_group=after_group,
offset_time=minute_clause % (1,),
min_time=minute_clause % (minutes + 1,),
max_time=minute_clause % (minutes * (60 / normalization),),
norm=normalization,
epoch_clause=epoch_clause,
now=now_clause,
project_ids=", ".join((str(int(x)) for x in project_ids)),
)
return RawQuerySet(self, query, params)
def query(self,
project,
query=None,
status=None,
tags=None,
bookmarked_by=None,
assigned_to=None,
first_release=None,
sort_by='date',
date_filter='last_seen',
unassigned=None,
date_from=None,
date_to=None,
cursor=None,
limit=100):
from sentry.models import Group, GroupStatus
queryset = Group.objects.filter(project=project)
if query:
# TODO(dcramer): if we want to continue to support search on SQL
# we should at least optimize this in Postgres so that it does
# the query filter **after** the index filters, and restricts the
# result set
queryset = queryset.filter(
Q(message__icontains=query) | Q(culprit__icontains=query))
if status is None:
queryset = queryset.exclude(status__in=(
GroupStatus.PENDING_DELETION,
GroupStatus.DELETION_IN_PROGRESS,
GroupStatus.PENDING_MERGE,
))
else:
queryset = queryset.filter(status=status)
if bookmarked_by:
queryset = queryset.filter(
bookmark_set__project=project,
bookmark_set__user=bookmarked_by,
)
if assigned_to:
queryset = queryset.filter(
assignee_set__project=project,
assignee_set__user=assigned_to,
)
elif unassigned in (True, False):
queryset = queryset.filter(assignee_set__isnull=unassigned, )
if first_release:
queryset = queryset.filter(
first_release__project=project,
first_release__version=first_release,
)
if tags:
for k, v in tags.iteritems():
queryset = queryset.filter(**dict(
grouptag__key=k,
grouptag__value=v,
))
if date_filter == 'first_seen':
if date_from and date_to:
queryset = queryset.filter(
first_seen__gte=date_from,
first_seen__lte=date_to,
)
elif date_from:
queryset = queryset.filter(first_seen__gte=date_from)
elif date_to:
queryset = queryset.filter(first_seen__lte=date_to)
elif date_filter == 'last_seen':
if date_from and date_to:
queryset = queryset.filter(
first_seen__gte=date_from,
last_seen__lte=date_to,
)
elif date_from:
queryset = queryset.filter(last_seen__gte=date_from)
elif date_to:
queryset = queryset.filter(last_seen__lte=date_to)
engine = get_db_engine('default')
if engine.startswith('sqlite'):
score_clause = SQLITE_SORT_CLAUSES[sort_by]
elif engine.startswith('mysql'):
score_clause = MYSQL_SORT_CLAUSES[sort_by]
elif engine.startswith('oracle'):
score_clause = ORACLE_SORT_CLAUSES[sort_by]
elif engine in MSSQL_ENGINES:
score_clause = MSSQL_SORT_CLAUSES[sort_by]
else:
score_clause = SORT_CLAUSES[sort_by]
if sort_by == 'tottime':
queryset = queryset.filter(time_spent_count__gt=0)
elif sort_by == 'avgtime':
queryset = queryset.filter(time_spent_count__gt=0)
queryset = queryset.extra(select={'sort_value': score_clause}, )
# HACK: don't sort by the same column twice
if sort_by == 'date':
paginator_cls = DateTimePaginator
sort_clause = '-last_seen'
elif sort_by == 'priority':
paginator_cls = Paginator
sort_clause = '-score'
elif sort_by == 'new':
paginator_cls = DateTimePaginator
sort_clause = '-first_seen'
elif sort_by == 'freq':
paginator_cls = Paginator
sort_clause = '-times_seen'
else:
paginator_cls = Paginator
sort_clause = '-sort_value'
queryset = queryset.order_by(sort_clause)
paginator = paginator_cls(queryset, sort_clause)
return paginator.get_result(limit, cursor)