def get_attrs(self, item_list, user, is_public=False):
eventstore.bind_nodes(item_list, "data")
crash_files = get_crash_files(item_list)
results = {}
for item in item_list:
# TODO(dcramer): convert to get_api_context
(user_data, user_meta) = self._get_interface_with_meta(
item, "user", is_public)
(contexts_data, contexts_meta) = self._get_interface_with_meta(
item, "contexts", is_public)
(sdk_data,
sdk_meta) = self._get_interface_with_meta(item, "sdk", is_public)
(entries, entries_meta) = self._get_entries(item,
user,
is_public=is_public)
crash_file = crash_files.get(item.event_id)
results[item] = {
"entries": entries,
"user": user_data,
"contexts": contexts_data or {},
"sdk": sdk_data,
"crash_file": serialize(crash_file, user=user),
"_meta": {
"entries": entries_meta,
"user": user_meta,
"contexts": contexts_meta,
"sdk": sdk_meta,
},
}
return results
def test_screams_bloody_murder_when_ref_fails(self):
project1 = self.create_project()
project2 = self.create_project()
invalid_event = self.store_event(
data={
"event_id": "a" * 32,
"timestamp": iso_format(before_now(minutes=1)),
"fingerprint": ["group-1"],
},
project_id=project1.id,
)
event = self.store_event(
data={
"event_id": "b" * 32,
"timestamp": iso_format(before_now(minutes=1)),
"fingerprint": ["group-2"],
},
project_id=project2.id,
)
event.data.bind_ref(invalid_event)
event.data.save()
assert event.data.get_ref(event) != event.data.get_ref(invalid_event)
# Unload node data to force reloading from nodestore
event.data._node_data = None
with pytest.raises(NodeIntegrityFailure):
eventstore.bind_nodes([event])
def test_event_node_id(self):
# Create an event without specifying node_id. A node_id should be generated
e1 = Event(project_id=1, event_id="abc", data={"foo": "bar"})
assert e1.data.id is not None, "We should have generated a node_id for this event"
e1_node_id = e1.data.id
e1.data.save()
e1_body = nodestore.get(e1_node_id)
assert e1_body == {
"foo": "bar"
}, "The event body should be in nodestore"
e1 = Event(project_id=1, event_id="abc")
assert e1.data.data == {
"foo": "bar"
}, "The event body should be loaded from nodestore"
assert e1.data.id == e1_node_id, "The event's node_id should be the same after load"
# Event with no data should not be saved to nodestore
e2 = Event(project_id=1, event_id="mno", data=None)
e2_node_id = e2.data.id
assert e2.data.data == {} # NodeData returns {} by default
eventstore.bind_nodes([e2], "data")
assert e2.data.data == {}
e2_body = nodestore.get(e2_node_id)
assert e2_body is None
def _attach_related(_events):
project_ids = set([event.project_id for event in _events])
projects = {
p.id: p
for p in Project.objects.filter(id__in=project_ids)
}
group_ids = set([event.group_id for event in _events])
groups = {g.id: g for g in Group.objects.filter(id__in=group_ids)}
for event in _events:
event.project = projects[event.project_id]
event.group = groups[event.group_id]
eventstore.bind_nodes(_events, "data")
def _attach_related(_events):
project_ids = set()
group_ids = set()
for event in _events:
project_ids.add(event.project_id)
group_ids.add(event.group_id)
projects = {p.id: p for p in Project.objects.filter(id__in=project_ids)}
groups = {g.id: g for g in Group.objects.filter(id__in=group_ids)}
for event in _events:
event.project = projects.get(event.project_id)
event.group = groups.get(event.group_id)
# When migrating old data from Sentry 9.0.0 to 9.1.2 to 10 in rapid succession, the event timestamp may be
# missing. This adds it back
if "timestamp" not in event.data.data:
event.data.data['timestamp'] = to_timestamp(event.datetime)
eventstore.bind_nodes(_events, "data")
def find_resolved(self, project_id, limit=100):
"""Returns a list of raw events that generally match the given
processing issue and no longer have any issues remaining. Returns
a list of raw events that are now resolved and a bool that indicates
if there are more.
"""
from sentry import eventstore
rv = list(self.find_resolved_queryset([project_id])[:limit])
if len(rv) > limit:
rv = rv[:limit]
has_more = True
else:
has_more = False
rv = list(rv)
eventstore.bind_nodes(rv, "data")
return rv, has_more
def unmerge(
project_id,
source_id,
destination_id,
fingerprints,
actor_id,
last_event=None,
batch_size=500,
source_fields_reset=False,
eventstream_state=None,
):
# XXX: The queryset chunking logic below is awfully similar to
# ``RangeQuerySetWrapper``. Ideally that could be refactored to be able to
# be run without iteration by passing around a state object and we could
# just use that here instead.
source = Group.objects.get(project_id=project_id, id=source_id)
# On the first iteration of this loop, we clear out all of the
# denormalizations from the source group so that we can have a clean slate
# for the new, repaired data.
if last_event is None:
fingerprints = lock_hashes(project_id, source_id, fingerprints)
truncate_denormalizations(source)
caches = get_caches()
project = caches["Project"](project_id)
# We process events sorted in descending order by -timestamp, -event_id. We need
# to include event_id as well as timestamp in the ordering criteria since:
#
# - Event timestamps are rounded to the second so multiple events are likely
# to have the same timestamp.
#
# - When sorting by timestamp alone, Snuba may not give us a deterministic
# order for events with the same timestamp.
#
# - We need to ensure that we do not skip any events between batches. If we
# only sorted by timestamp < last_event.timestamp it would be possible to
# have missed an event with the same timestamp as the last item in the
# previous batch.
conditions = []
if last_event is not None:
conditions.extend([
["timestamp", "<=", last_event["timestamp"]],
[
["timestamp", "<", last_event["timestamp"]],
["event_id", "<", last_event["event_id"]],
],
])
events = eventstore.get_events(
filter=eventstore.Filter(project_ids=[project_id],
group_ids=[source.id],
conditions=conditions),
limit=batch_size,
referrer="unmerge",
orderby=["-timestamp", "-event_id"],
)
# If there are no more events to process, we're done with the migration.
if not events:
unlock_hashes(project_id, fingerprints)
logger.warning("Unmerge complete (eventstream state: %s)",
eventstream_state)
if eventstream_state:
eventstream.end_unmerge(eventstream_state)
return destination_id
if not options.get("eventstore.use-nodestore"):
eventstore.bind_nodes(events, "data")
source_events = []
destination_events = []
for event in events:
(destination_events if get_fingerprint(event) in fingerprints else
source_events).append(event)
if source_events:
if not source_fields_reset:
source.update(
**get_group_creation_attributes(caches, source_events))
source_fields_reset = True
else:
source.update(
**get_group_backfill_attributes(caches, source, source_events))
(destination_id, eventstream_state) = migrate_events(
caches,
project,
source_id,
destination_id,
fingerprints,
destination_events,
actor_id,
eventstream_state,
)
repair_denormalizations(caches, project, events)
unmerge.delay(
project_id,
source_id,
destination_id,
fingerprints,
actor_id,
last_event={
"timestamp": events[-1].timestamp,
"event_id": events[-1].event_id
},
batch_size=batch_size,
source_fields_reset=source_fields_reset,
eventstream_state=eventstream_state,
)
