def wait_for_event_count(self, project_id, total, attempts=2):
"""
Wait until the event count reaches the provided value or until attempts is reached.
Useful when you're storing several events and need to ensure that snuba/clickhouse
state has settled.
"""
# Verify that events have settled in snuba's storage.
# While snuba is synchronous, clickhouse isn't entirely synchronous.
attempt = 0
snuba_filter = eventstore.Filter(project_ids=[project_id])
while attempt < attempts:
events = eventstore.get_events(snuba_filter)
if len(events) >= total:
break
attempt += 1
time.sleep(0.05)
if attempt == attempts:
assert False, "Could not ensure event was persisted within {} attempt(s)".format(
attempt)
def test_reprocessing(self):
self.project.update_option("sentry:store_crash_reports", STORE_CRASH_REPORTS_ALL)
with self.feature(
{"organizations:event-attachments": True, "projects:reprocessing-v2": True}
):
with open(get_fixture_path("windows.dmp"), "rb") as f:
event = self.post_and_retrieve_minidump(
{"upload_file_minidump": f}, {"sentry[logger]": "test-logger"}
)
insta_snapshot_stacktrace_data(self, event.data, subname="initial")
self.upload_symbols()
from sentry.tasks.reprocessing2 import reprocess_event
with self.tasks():
reprocess_event.delay(
project_id=self.project.id, event_id=event.event_id, start_time=time.time()
)
(new_event,) = eventstore.get_events(
eventstore.Filter(
project_ids=[self.project.id],
conditions=[["tags[original_event_id]", "=", event.event_id]],
)
)
assert new_event is not None
assert new_event.event_id != event.event_id
insta_snapshot_stacktrace_data(self, new_event.data, subname="reprocessed")
for event_id in (event.event_id, new_event.event_id):
(minidump,) = sorted(
EventAttachment.objects.filter(event_id=new_event.event_id), key=lambda x: x.name
)
assert minidump.name == "windows.dmp"
assert minidump.file.type == "event.minidump"
assert minidump.file.checksum == "74bb01c850e8d65d3ffbc5bad5cabc4668fce247"
def assertReportCreated(self, input, output):
resp = self._postCspWithHeader(input)
assert resp.status_code == 201, resp.content
# XXX: there appears to be a race condition between the 201 return and get_events,
# leading this test to sometimes fail. .5s seems to be sufficient.
# Modifying the timestamp of store_event, like how it's done in other snuba tests,
# doesn't work here because the event isn't created directly by this test.
sleep(0.5)
events = eventstore.get_events(
filter=eventstore.Filter(
project_ids=[self.project.id], conditions=[["type", "=", "csp"]]
)
)
assert len(events) == 1
e = events[0]
e.bind_node_data()
assert output["message"] == e.data["logentry"]["formatted"]
for key, value in six.iteritems(output["tags"]):
assert e.get_tag(key) == value
for key, value in six.iteritems(output["data"]):
assert e.data[key] == value
def get_oldest_or_latest_event_for_environments(
ordering, environments=(), issue_id=None, project_id=None
):
conditions = []
if len(environments) > 0:
conditions.append(["environment", "IN", environments])
events = eventstore.get_events(
filter=eventstore.Filter(
conditions=conditions, project_ids=[project_id], group_ids=[issue_id]
),
limit=1,
orderby=ordering.value,
referrer="Group.get_latest",
)
if events:
return events[0]
return None
def chunk(self):
conditions = []
if self.last_event is not None:
conditions.extend([
["timestamp", "<=", self.last_event.timestamp],
[
["timestamp", "<", self.last_event.timestamp],
["event_id", "<", self.last_event.event_id],
],
])
events = eventstore.get_events(
filter=eventstore.Filter(conditions=conditions,
project_ids=[self.project_id],
group_ids=[self.group_id]),
limit=self.DEFAULT_CHUNK_SIZE,
referrer="deletions.group",
orderby=["-timestamp", "-event_id"],
)
if not events:
return False
self.last_event = events[-1]
# Remove from nodestore
node_ids = [
Event.generate_node_id(self.project_id, event.event_id)
for event in events
]
nodestore.delete_multi(node_ids)
# Remove EventAttachment and UserReport
event_ids = [event.event_id for event in events]
EventAttachment.objects.filter(event_id__in=event_ids,
project_id=self.project_id).delete()
UserReport.objects.filter(event_id__in=event_ids,
project_id=self.project_id).delete()
return True
def get(self, request, organization, event_id):
"""
Resolve an Event ID
``````````````````
This resolves an event ID to the project slug and internal issue ID and internal event ID.
:pparam string organization_slug: the slug of the organization the
event ID should be looked up in.
:param string event_id: the event ID to look up.
:auth: required
"""
# Largely copied from ProjectGroupIndexEndpoint
if len(event_id) != 32:
return Response({"detail": "Event ID must be 32 characters."}, status=400)
project_slugs_by_id = dict(
Project.objects.filter(organization=organization).values_list("id", "slug")
)
try:
snuba_filter = eventstore.Filter(
conditions=[["event.type", "!=", "transaction"]],
project_ids=list(project_slugs_by_id.keys()),
event_ids=[event_id],
)
event = eventstore.get_events(filter=snuba_filter, limit=1)[0]
except IndexError:
raise ResourceDoesNotExist()
else:
return Response(
{
"organizationSlug": organization.slug,
"projectSlug": project_slugs_by_id[event.project_id],
"groupId": six.text_type(event.group_id),
"eventId": six.text_type(event.event_id),
"event": serialize(event, request.user),
}
)
def get(self, request, organization, event_id):
"""
Resolve an Event ID
``````````````````
This resolves an event ID to the project slug and internal issue ID and internal event ID.
:pparam string organization_slug: the slug of the organization the
event ID should be looked up in.
:param string event_id: the event ID to look up. validated by a
regex in the URL.
:auth: required
"""
project_slugs_by_id = dict(
Project.objects.filter(organization=organization).values_list(
"id", "slug"))
try:
snuba_filter = eventstore.Filter(
conditions=[["event.type", "!=", "transaction"]],
project_ids=list(project_slugs_by_id.keys()),
event_ids=[event_id],
)
event = eventstore.get_events(filter=snuba_filter, limit=1)[0]
except IndexError:
raise ResourceDoesNotExist()
else:
return Response({
"organizationSlug":
organization.slug,
"projectSlug":
project_slugs_by_id[event.project_id],
"groupId":
str(event.group_id),
"eventId":
str(event.event_id),
"event":
serialize(event, request.user),
})
def unreal_crash_test_impl(self, filename):
self.project.update_option("sentry:store_crash_reports", True)
self.upload_symbols()
# attachments feature has to be on for the files extract stick around
with self.feature("organizations:event-attachments"):
with open(filename, "rb") as f:
resp = self._postUnrealWithHeader(f.read())
assert resp.status_code == 200
event = eventstore.get_events(
filter_keys={"project_id": [self.project.id]})[0]
self.insta_snapshot({
"contexts": event.data.get("contexts"),
"exception": event.data.get("exception"),
"stacktrace": event.data.get("stacktrace"),
"threads": event.data.get("threads"),
"extra": event.data.get("extra"),
})
return sorted(EventAttachment.objects.filter(event_id=event.event_id),
key=lambda x: x.name)
def post_and_retrieve_security_report(self, data):
url = self.get_relay_security_url(self.project.id, self.projectkey.public_key)
responses.add_passthru(url)
event_ids = {
event.event_id
for event in eventstore.get_events(eventstore.Filter(project_ids=[self.project.id]))
}
def has_new_event():
# Hack: security report endpoint does not return event ID
for event in eventstore.get_events(eventstore.Filter(project_ids=[self.project.id])):
if event.event_id not in event_ids:
return event
resp = requests.post(url, json=data)
assert resp.ok
event = self.wait_for_ingest_consumer(has_new_event)
# check that we found it in Snuba
assert event
return event
def _get_terminal_event_id(self, direction, snuba_args, event):
if direction == Direction.NEXT:
time_condition = [["timestamp", ">", event.timestamp]]
orderby = ["-timestamp", "-event_id"]
else:
time_condition = [["timestamp", "<", event.timestamp]]
orderby = ["timestamp", "event_id"]
conditions = snuba_args["conditions"][:]
conditions.extend(time_condition)
result = eventstore.get_events(
start=snuba_args.get("start", None),
end=snuba_args.get("end", None),
conditions=conditions,
filter_keys=snuba_args["filter_keys"],
orderby=orderby,
limit=1,
)
if not result:
return None
return result[0].event_id
def test_concurrent_events_go_into_new_group(default_project, reset_snuba,
register_event_preprocessor,
process_and_save,
burst_task_runner):
@register_event_preprocessor
def event_preprocessor(data):
extra = data.setdefault("extra", {})
extra.setdefault("processing_counter", 0)
extra["processing_counter"] += 1
return data
event_id = process_and_save({"message": "hello world"})
event = eventstore.get_event_by_id(default_project.id, event_id)
with burst_task_runner() as burst_reprocess:
reprocess_group(default_project.id, event.group_id)
assert not is_group_finished(event.group_id)
event_id2 = process_and_save({"message": "hello world"})
event2 = eventstore.get_event_by_id(default_project.id, event_id2)
assert event2.event_id != event.event_id
assert event2.group_id != event.group_id
burst_reprocess()
(event3, ) = eventstore.get_events(
eventstore.Filter(
project_ids=[default_project.id],
conditions=[["tags[original_event_id]", "=", event_id]],
))
assert is_group_finished(event.group_id)
assert event2.group_id == event3.group_id
assert event.get_hashes() == event2.get_hashes() == event3.get_hashes()
def test_basic_resolving(self):
url = reverse(
'sentry-api-0-dsym-files',
kwargs={
'organization_slug': self.project.organization.slug,
'project_slug': self.project.slug,
}
)
self.login_as(user=self.user)
out = BytesIO()
f = zipfile.ZipFile(out, 'w')
f.writestr('proguard/%s.txt' % PROGUARD_UUID, PROGUARD_SOURCE)
f.writestr('ignored-file.txt', b'This is just some stuff')
f.close()
response = self.client.post(
url, {
'file':
SimpleUploadedFile(
'symbols.zip',
out.getvalue(),
content_type='application/zip'),
},
format='multipart'
)
assert response.status_code == 201, response.content
assert len(response.data) == 1
event_data = {
"user": {
"ip_address": "31.172.207.97"
},
"extra": {},
"project": self.project.id,
"platform": "java",
"debug_meta": {
"images": [{
"type": "proguard",
"uuid": PROGUARD_UUID,
}]
},
"exception": {
"values": [
{
'stacktrace': {
"frames": [
{
"function": "a",
"abs_path": None,
"module": "org.a.b.g$a",
"filename": None,
"lineno": 67,
},
{
"function": "a",
"abs_path": None,
"module": "org.a.b.g$a",
"filename": None,
"lineno": 69,
},
]
},
"type": "RuntimeException",
"value": "Shit broke yo"
}
]
},
}
# We do a preflight post, because there are many queries polluting the array
# before the actual "processing" happens (like, auth_user)
self._postWithHeader(event_data)
with self.assertWriteQueries({
'nodestore_node': 2,
'sentry_eventtag': 1,
'sentry_eventuser': 1,
'sentry_filtervalue': 2,
'sentry_groupedmessage': 1,
'sentry_message': 1,
'sentry_messagefiltervalue': 2,
'sentry_userip': 1,
'sentry_userreport': 1
}):
resp = self._postWithHeader(event_data)
assert resp.status_code == 200
event = eventstore.get_events(filter_keys={'project_id': [self.project.id]})[0]
bt = event.interfaces['exception'].values[0].stacktrace
frames = bt.frames
assert frames[0].function == 'getClassContext'
assert frames[0].module == 'org.slf4j.helpers.Util$ClassContextSecurityManager'
assert frames[1].function == 'getExtraClassContext'
assert frames[1].module == 'org.slf4j.helpers.Util$ClassContextSecurityManager'
assert event.culprit == (
'org.slf4j.helpers.Util$ClassContextSecurityManager '
'in getExtraClassContext'
)
def test_error_on_resolving(self):
url = reverse(
'sentry-api-0-dsym-files',
kwargs={
'organization_slug': self.project.organization.slug,
'project_slug': self.project.slug,
}
)
self.login_as(user=self.user)
out = BytesIO()
f = zipfile.ZipFile(out, 'w')
f.writestr('proguard/%s.txt' % PROGUARD_BUG_UUID, PROGUARD_BUG_SOURCE)
f.close()
response = self.client.post(
url, {
'file':
SimpleUploadedFile('symbols.zip', out.getvalue(),
content_type='application/zip'),
},
format='multipart'
)
assert response.status_code == 201, response.content
assert len(response.data) == 1
event_data = {
"user": {
"ip_address": "31.172.207.97"
},
"extra": {},
"project": self.project.id,
"platform": "java",
"debug_meta": {
"images": [{
"type": "proguard",
"uuid": PROGUARD_BUG_UUID,
}]
},
"exception": {
"values": [
{
'stacktrace': {
"frames": [
{
"function": "a",
"abs_path": None,
"module": "org.a.b.g$a",
"filename": None,
"lineno": 67,
},
{
"function": "a",
"abs_path": None,
"module": "org.a.b.g$a",
"filename": None,
"lineno": 69,
},
]
},
"type": "RuntimeException",
"value": "Shit broke yo"
}
]
},
}
resp = self._postWithHeader(event_data)
assert resp.status_code == 200
event = eventstore.get_events(filter_keys={'project_id': [self.project.id]})[0]
assert len(event.data['errors']) == 1
assert event.data['errors'][0] == {
'mapping_uuid': u'071207ac-b491-4a74-957c-2c94fd9594f2',
'type': 'proguard_missing_lineno',
}
def unmerge(
project_id,
source_id,
destination_id,
fingerprints,
actor_id,
last_event=None,
batch_size=500,
source_fields_reset=False,
eventstream_state=None,
):
# XXX: The queryset chunking logic below is awfully similar to
# ``RangeQuerySetWrapper``. Ideally that could be refactored to be able to
# be run without iteration by passing around a state object and we could
# just use that here instead.
source = Group.objects.get(project_id=project_id, id=source_id)
# On the first iteration of this loop, we clear out all of the
# denormalizations from the source group so that we can have a clean slate
# for the new, repaired data.
if last_event is None:
fingerprints = lock_hashes(project_id, source_id, fingerprints)
truncate_denormalizations(source)
caches = get_caches()
project = caches["Project"](project_id)
# We process events sorted in descending order by -timestamp, -event_id. We need
# to include event_id as well as timestamp in the ordering criteria since:
#
# - Event timestamps are rounded to the second so multiple events are likely
# to have the same timestamp.
#
# - When sorting by timestamp alone, Snuba may not give us a deterministic
# order for events with the same timestamp.
#
# - We need to ensure that we do not skip any events between batches. If we
# only sorted by timestamp < last_event.timestamp it would be possible to
# have missed an event with the same timestamp as the last item in the
# previous batch.
conditions = []
if last_event is not None:
conditions.extend(
[
["timestamp", "<=", last_event["timestamp"]],
[
["timestamp", "<", last_event["timestamp"]],
["event_id", "<", last_event["event_id"]],
],
]
)
events = eventstore.get_events(
filter_keys={"project_id": [project_id], "issue": [source.id]},
# We need the text-only "search message" from Snuba, not the raw message
# dict field from nodestore.
additional_columns=[eventstore.Columns.MESSAGE],
conditions=conditions,
limit=batch_size,
referrer="unmerge",
orderby=["-timestamp", "-event_id"],
)
# If there are no more events to process, we're done with the migration.
if not events:
tagstore.update_group_tag_key_values_seen(project_id, [source_id, destination_id])
unlock_hashes(project_id, fingerprints)
logger.warning("Unmerge complete (eventstream state: %s)", eventstream_state)
if eventstream_state:
eventstream.end_unmerge(eventstream_state)
return destination_id
Event.objects.bind_nodes(events, "data")
source_events = []
destination_events = []
for event in events:
(destination_events if get_fingerprint(event) in fingerprints else source_events).append(
event
)
if source_events:
if not source_fields_reset:
source.update(**get_group_creation_attributes(caches, source_events))
source_fields_reset = True
else:
source.update(**get_group_backfill_attributes(caches, source, source_events))
(destination_id, eventstream_state) = migrate_events(
caches,
project,
source_id,
destination_id,
fingerprints,
destination_events,
actor_id,
eventstream_state,
)
repair_denormalizations(caches, project, events)
unmerge.delay(
project_id,
source_id,
destination_id,
fingerprints,
actor_id,
last_event={"timestamp": events[-1].timestamp, "event_id": events[-1].event_id},
batch_size=batch_size,
source_fields_reset=source_fields_reset,
eventstream_state=eventstream_state,
)
def test_sourcemap_expansion(self):
responses.add(responses.GET,
'http://example.com/test.js',
body=load_fixture('test.js'),
content_type='application/javascript')
responses.add(responses.GET,
'http://example.com/test.min.js',
body=load_fixture('test.min.js'),
content_type='application/javascript')
responses.add(responses.GET,
'http://example.com/test.map',
body=load_fixture('test.map'),
content_type='application/json')
responses.add(responses.GET,
'http://example.com/index.html',
body='Not Found',
status=404)
data = {
'message': 'hello',
'platform': 'javascript',
'exception': {
'values': [{
'type': 'Error',
'stacktrace': {
'frames':
json.loads(load_fixture('minifiedError.json'))[::-1],
},
}],
}
}
resp = self._postWithHeader(data)
assert resp.status_code == 200
event = eventstore.get_events(
filter_keys={'project_id': [self.project.id]})[0]
exception = event.interfaces['exception']
frame_list = exception.values[0].stacktrace.frames
assert len(frame_list) == 4
import pprint
pprint.pprint(frame_list)
assert frame_list[0].function == 'produceStack'
assert frame_list[0].lineno == 6
assert frame_list[0].filename == 'index.html'
assert frame_list[1].function == 'test'
assert frame_list[1].lineno == 20
assert frame_list[1].filename == 'test.js'
assert frame_list[2].function == 'invoke'
assert frame_list[2].lineno == 15
assert frame_list[2].filename == 'test.js'
assert frame_list[3].function == 'onFailure'
assert frame_list[3].lineno == 5
assert frame_list[3].filename == 'test.js'
def has_new_event():
# Hack: security report endpoint does not return event ID
for event in eventstore.get_events(eventstore.Filter(project_ids=[self.project.id])):
if event.event_id not in event_ids:
return event
def serialize(self,
parent_map,
root,
warning_extra,
params,
snuba_event=None,
event_id=None):
""" For the full event trace, we return the results as a graph instead of a flattened list """
parent_events = {}
result = parent_events[root["id"]] = self.serialize_event(
root, None, 0, True)
event_ids = []
for events in parent_map.values():
event_ids.extend(event["id"] for event in events)
with sentry_sdk.start_span(
op="nodestore",
description=f"retrieving {len(parent_map)} nodes") as span:
span.set_data("total nodes", len(parent_map))
node_data = {
event.event_id: event
for event in eventstore.get_events(
eventstore.Filter(
project_ids=params["project_id"],
event_ids=event_ids,
))
}
with sentry_sdk.start_span(op="building.trace",
description="full trace"):
to_check = deque([root])
iteration = 0
while to_check:
current_event = to_check.popleft()
event = node_data.get(current_event["id"])
previous_event = parent_events[current_event["id"]]
previous_event.update({
event_key: event.data.get(event_key)
for event_key in NODESTORE_KEYS
})
for child in event.data.get("spans", []):
if child["span_id"] not in parent_map:
continue
# Avoid potential span loops by popping, so we don't traverse the same nodes twice
child_events = parent_map.pop(child["span_id"])
for child_event in child_events:
parent_events[
child_event["id"]] = self.serialize_event(
child_event, current_event["id"],
previous_event["generation"] + 1)
# Add this event to its parent's children
previous_event["children"].append(
parent_events[child_event["id"]])
to_check.append(child_event)
# Limit iterations just to be safe
iteration += 1
if iteration > MAX_TRACE_SIZE:
logger.warning(
"discover.trace-view.surpassed-trace-limit",
extra=warning_extra,
)
break
return result
def test_sourcemap_expansion(self):
responses.add(
responses.GET,
"http://example.com/test.js",
body=load_fixture("test.js"),
content_type="application/javascript",
)
responses.add(
responses.GET,
"http://example.com/test.min.js",
body=load_fixture("test.min.js"),
content_type="application/javascript",
)
responses.add(
responses.GET,
"http://example.com/test.map",
body=load_fixture("test.map"),
content_type="application/json",
)
responses.add(responses.GET,
"http://example.com/index.html",
body="Not Found",
status=404)
min_ago = iso_format(before_now(minutes=1))
data = {
"timestamp": min_ago,
"message": "hello",
"platform": "javascript",
"exception": {
"values": [{
"type": "Error",
"stacktrace": {
"frames":
json.loads(load_fixture("minifiedError.json"))[::-1]
},
}]
},
}
resp = self._postWithHeader(data)
assert resp.status_code == 200
event = eventstore.get_events(
filter_keys={"project_id": [self.project.id]})[0]
exception = event.interfaces["exception"]
frame_list = exception.values[0].stacktrace.frames
assert len(frame_list) == 4
import pprint
pprint.pprint(frame_list)
assert frame_list[0].function == "produceStack"
assert frame_list[0].lineno == 6
assert frame_list[0].filename == "index.html"
assert frame_list[1].function == "test"
assert frame_list[1].lineno == 20
assert frame_list[1].filename == "test.js"
assert frame_list[2].function == "invoke"
assert frame_list[2].lineno == 15
assert frame_list[2].filename == "test.js"
assert frame_list[3].function == "onFailure"
assert frame_list[3].lineno == 5
assert frame_list[3].filename == "test.js"
def get_event(self):
return eventstore.get_events(filter_keys={"project_id": [self.project.id]})[0]
