def get_attrs(self, item_list, user, is_public=False): eventstore.bind_nodes(item_list, "data") crash_files = get_crash_files(item_list) results = {} for item in item_list: # TODO(dcramer): convert to get_api_context (user_data, user_meta) = self._get_interface_with_meta( item, "user", is_public) (contexts_data, contexts_meta) = self._get_interface_with_meta( item, "contexts", is_public) (sdk_data, sdk_meta) = self._get_interface_with_meta(item, "sdk", is_public) (entries, entries_meta) = self._get_entries(item, user, is_public=is_public) crash_file = crash_files.get(item.event_id) results[item] = { "entries": entries, "user": user_data, "contexts": contexts_data or {}, "sdk": sdk_data, "crash_file": serialize(crash_file, user=user), "_meta": { "entries": entries_meta, "user": user_meta, "contexts": contexts_meta, "sdk": sdk_meta, }, } return results
def test_screams_bloody_murder_when_ref_fails(self): project1 = self.create_project() project2 = self.create_project() invalid_event = self.store_event( data={ "event_id": "a" * 32, "timestamp": iso_format(before_now(minutes=1)), "fingerprint": ["group-1"], }, project_id=project1.id, ) event = self.store_event( data={ "event_id": "b" * 32, "timestamp": iso_format(before_now(minutes=1)), "fingerprint": ["group-2"], }, project_id=project2.id, ) event.data.bind_ref(invalid_event) event.data.save() assert event.data.get_ref(event) != event.data.get_ref(invalid_event) # Unload node data to force reloading from nodestore event.data._node_data = None with pytest.raises(NodeIntegrityFailure): eventstore.bind_nodes([event])
def test_event_node_id(self): # Create an event without specifying node_id. A node_id should be generated e1 = Event(project_id=1, event_id="abc", data={"foo": "bar"}) assert e1.data.id is not None, "We should have generated a node_id for this event" e1_node_id = e1.data.id e1.data.save() e1_body = nodestore.get(e1_node_id) assert e1_body == { "foo": "bar" }, "The event body should be in nodestore" e1 = Event(project_id=1, event_id="abc") assert e1.data.data == { "foo": "bar" }, "The event body should be loaded from nodestore" assert e1.data.id == e1_node_id, "The event's node_id should be the same after load" # Event with no data should not be saved to nodestore e2 = Event(project_id=1, event_id="mno", data=None) e2_node_id = e2.data.id assert e2.data.data == {} # NodeData returns {} by default eventstore.bind_nodes([e2], "data") assert e2.data.data == {} e2_body = nodestore.get(e2_node_id) assert e2_body is None
def _attach_related(_events): project_ids = set([event.project_id for event in _events]) projects = { p.id: p for p in Project.objects.filter(id__in=project_ids) } group_ids = set([event.group_id for event in _events]) groups = {g.id: g for g in Group.objects.filter(id__in=group_ids)} for event in _events: event.project = projects[event.project_id] event.group = groups[event.group_id] eventstore.bind_nodes(_events, "data")
def _attach_related(_events): project_ids = set() group_ids = set() for event in _events: project_ids.add(event.project_id) group_ids.add(event.group_id) projects = {p.id: p for p in Project.objects.filter(id__in=project_ids)} groups = {g.id: g for g in Group.objects.filter(id__in=group_ids)} for event in _events: event.project = projects.get(event.project_id) event.group = groups.get(event.group_id) # When migrating old data from Sentry 9.0.0 to 9.1.2 to 10 in rapid succession, the event timestamp may be # missing. This adds it back if "timestamp" not in event.data.data: event.data.data['timestamp'] = to_timestamp(event.datetime) eventstore.bind_nodes(_events, "data")
def find_resolved(self, project_id, limit=100): """Returns a list of raw events that generally match the given processing issue and no longer have any issues remaining. Returns a list of raw events that are now resolved and a bool that indicates if there are more. """ from sentry import eventstore rv = list(self.find_resolved_queryset([project_id])[:limit]) if len(rv) > limit: rv = rv[:limit] has_more = True else: has_more = False rv = list(rv) eventstore.bind_nodes(rv, "data") return rv, has_more
def unmerge( project_id, source_id, destination_id, fingerprints, actor_id, last_event=None, batch_size=500, source_fields_reset=False, eventstream_state=None, ): # XXX: The queryset chunking logic below is awfully similar to # ``RangeQuerySetWrapper``. Ideally that could be refactored to be able to # be run without iteration by passing around a state object and we could # just use that here instead. source = Group.objects.get(project_id=project_id, id=source_id) # On the first iteration of this loop, we clear out all of the # denormalizations from the source group so that we can have a clean slate # for the new, repaired data. if last_event is None: fingerprints = lock_hashes(project_id, source_id, fingerprints) truncate_denormalizations(source) caches = get_caches() project = caches["Project"](project_id) # We process events sorted in descending order by -timestamp, -event_id. We need # to include event_id as well as timestamp in the ordering criteria since: # # - Event timestamps are rounded to the second so multiple events are likely # to have the same timestamp. # # - When sorting by timestamp alone, Snuba may not give us a deterministic # order for events with the same timestamp. # # - We need to ensure that we do not skip any events between batches. If we # only sorted by timestamp < last_event.timestamp it would be possible to # have missed an event with the same timestamp as the last item in the # previous batch. conditions = [] if last_event is not None: conditions.extend([ ["timestamp", "<=", last_event["timestamp"]], [ ["timestamp", "<", last_event["timestamp"]], ["event_id", "<", last_event["event_id"]], ], ]) events = eventstore.get_events( filter=eventstore.Filter(project_ids=[project_id], group_ids=[source.id], conditions=conditions), limit=batch_size, referrer="unmerge", orderby=["-timestamp", "-event_id"], ) # If there are no more events to process, we're done with the migration. if not events: unlock_hashes(project_id, fingerprints) logger.warning("Unmerge complete (eventstream state: %s)", eventstream_state) if eventstream_state: eventstream.end_unmerge(eventstream_state) return destination_id if not options.get("eventstore.use-nodestore"): eventstore.bind_nodes(events, "data") source_events = [] destination_events = [] for event in events: (destination_events if get_fingerprint(event) in fingerprints else source_events).append(event) if source_events: if not source_fields_reset: source.update( **get_group_creation_attributes(caches, source_events)) source_fields_reset = True else: source.update( **get_group_backfill_attributes(caches, source, source_events)) (destination_id, eventstream_state) = migrate_events( caches, project, source_id, destination_id, fingerprints, destination_events, actor_id, eventstream_state, ) repair_denormalizations(caches, project, events) unmerge.delay( project_id, source_id, destination_id, fingerprints, actor_id, last_event={ "timestamp": events[-1].timestamp, "event_id": events[-1].event_id }, batch_size=batch_size, source_fields_reset=source_fields_reset, eventstream_state=eventstream_state, )