def backfill_eventstream(apps, schema_editor): """ Inserts Postgres events into the eventstream if there are recent events in Postgres. This is for open source users migrating from 9.x who want to keep their events. If there are no recent events in Postgres, skip the backfill. """ from sentry import eventstore, eventstream from sentry.utils.query import RangeQuerySetWrapper Event = apps.get_model("sentry", "Event") Group = apps.get_model("sentry", "Group") Project = apps.get_model("sentry", "Project") # Kill switch to skip this migration skip_backfill = os.environ.get("SENTRY_SKIP_EVENTS_BACKFILL_FOR_10", False) # Use 90 day retention if the option has not been set or set to 0 DEFAULT_RETENTION = 90 retention_days = options.get( "system.event-retention-days") or DEFAULT_RETENTION def get_events(last_days): to_date = timezone.now() from_date = to_date - timedelta(days=last_days) return Event.objects.filter(datetime__gte=from_date, datetime__lte=to_date, group_id__isnull=False) def _attach_related(_events): project_ids = set() group_ids = set() for event in _events: project_ids.add(event.project_id) group_ids.add(event.group_id) projects = { p.id: p for p in Project.objects.filter(id__in=project_ids) } groups = {g.id: g for g in Group.objects.filter(id__in=group_ids)} for event in _events: event.project = projects.get(event.project_id) event.group = groups.get(event.group_id) eventstore.bind_nodes(_events, "data") if skip_backfill: print("Skipping backfill.\n") return events = get_events(retention_days) count = events.count() if count == 0: print("Nothing to do, skipping migration.\n") return print("Events to process: {}\n".format(count)) processed = 0 for e in RangeQuerySetWrapper(events, step=100, callbacks=(_attach_related, )): event = NewEvent(project_id=e.project_id, event_id=e.event_id, group_id=e.group_id, data=e.data.data) primary_hash = event.get_primary_hash() if event.project is None or event.group is None or len( event.data) == 0: print( "Skipped {} as group, project or node data information is invalid.\n" .format(event)) continue try: eventstream.insert( group=event.group, event=event, is_new=False, is_regression=False, is_new_group_environment=False, primary_hash=primary_hash, received_timestamp=event.data.get("received") or float(event.datetime.strftime("%s")), skip_consume=True, ) processed += 1 except Exception as error: print( "An error occured while trying to instert the following event: {}\n.----\n{}" .format(event, error)) if processed == 0: raise Exception( "Cannot migrate any event. If this is okay, re-run migrations with SENTRY_SKIP_EVENTS_BACKFILL_FOR_10 environment variable set to skip this step." ) print("Event migration done. Migrated {} of {} events.\n".format( processed, count))
def backfill_eventstream(apps, schema_editor): """ Inserts Postgres events into the eventstream if there are recent events in Postgres. This is for open source users migrating from 9.x who want to keep their events. If there are no recent events in Postgres, skip the backfill. """ from sentry import eventstore, eventstream from sentry.utils.query import RangeQuerySetWrapper Event = apps.get_model("sentry", "Event") Group = apps.get_model("sentry", "Group") Project = apps.get_model("sentry", "Project") # Kill switch to skip this migration skip_backfill = os.environ.get("SENTRY_SKIP_EVENTS_BACKFILL_FOR_10", False) # Use 90 day retention if the option has not been set or set to 0 DEFAULT_RETENTION = 90 retention_days = options.get( "system.event-retention-days") or DEFAULT_RETENTION def get_events(last_days): to_date = timezone.now() from_date = to_date - timedelta(days=last_days) return Event.objects.filter(datetime__gte=from_date, datetime__lte=to_date, group_id__isnull=False) def _attach_related(_events): project_ids = set() group_ids = set() for event in _events: project_ids.add(event.project_id) group_ids.add(event.group_id) projects = { p.id: p for p in Project.objects.filter(id__in=project_ids) } groups = {g.id: g for g in Group.objects.filter(id__in=group_ids)} for event in _events: event.project = projects.get(event.project_id) event.group = groups.get(event.group_id) # When migrating old data from Sentry 9.0.0 to 9.1.2 to 10 in rapid succession, the event timestamp may be # missing. This adds it back if "timestamp" not in event.data.data: event.data.data["timestamp"] = to_timestamp(event.datetime) eventstore.bind_nodes(_events, "data") if skip_backfill: print("Skipping backfill.\n") # noqa: B314 return events = get_events(retention_days) count = events.count() if count == 0: print("Nothing to do, skipping migration.\n") # noqa: B314 return print("Events to process: {}\n".format(count)) # noqa: B314 processed = 0 for e in RangeQuerySetWrapper(events, step=100, callbacks=(_attach_related, )): event = NewEvent(project_id=e.project_id, event_id=e.event_id, group_id=e.group_id, data=e.data.data) try: group = event.group except Group.DoesNotExist: group = None if event.project is None or group is None or len(event.data) == 0: print( # noqa: B314 "Skipped {} as group, project or node data information is invalid.\n" .format(event)) continue try: eventstream.insert( group=event.group, event=event, is_new=False, is_regression=False, is_new_group_environment=False, primary_hash=event.get_primary_hash(), received_timestamp=event.data.get("received") or float(event.datetime.strftime("%s")), skip_consume=True, ) # The node ID format was changed in Sentry 9.1.0 # (https://github.com/getsentry/sentry/commit/f73a4039d16a5c4f88bde37f6464cac21deb50e1) # If we are migrating from older versions of Sentry (i.e. 9.0.0 and earlier) # we need to resave the node using the new node ID scheme and delete the old # node. old_node_id = e.data.id new_node_id = event.data.id if old_node_id != new_node_id: event.data.save() nodestore.delete(old_node_id) processed += 1 except Exception as error: print( # noqa: B314 "An error occured while trying to migrate the following event: {}\n.----\n{}" .format(event, error)) if processed == 0: raise Exception( "Cannot migrate any event. If this is okay, re-run migrations with SENTRY_SKIP_EVENTS_BACKFILL_FOR_10 environment variable set to skip this step." ) print( # noqa: B314 "Event migration done. Migrated {} of {} events.\n".format( processed, count))