def merge_group(from_object_id=None, to_object_id=None, transaction_id=None, recursed=False, **kwargs): # TODO(mattrobenolt): Write tests for all of this from sentry.models import ( Activity, Group, GroupAssignee, GroupHash, GroupRuleStatus, GroupSubscription, GroupTagKey, GroupTagValue, EventMapping, Event, UserReport, GroupRedirect, GroupMeta, ) if not (from_object_id and to_object_id): logger.error('group.malformed.missing_params', extra={ 'transaction_id': transaction_id, }) return try: group = Group.objects.get(id=from_object_id) except Group.DoesNotExist: logger.warn('group.malformed.invalid_id', extra={ 'transaction_id': transaction_id, 'old_object_id': from_object_id, }) return try: new_group = Group.objects.get(id=to_object_id) except Group.DoesNotExist: logger.warn('group.malformed.invalid_id', extra={ 'transaction_id': transaction_id, 'old_object_id': from_object_id, }) return if not recursed: logger.info( 'merge.queued', extra={ 'transaction_id': transaction_id, 'new_group_id': new_group.id, 'old_group_id': group.id, 'new_event_id': getattr( new_group.event_set.order_by('-id').first(), 'id', None), 'old_event_id': getattr(group.event_set.order_by('-id').first(), 'id', None), 'new_hash_id': getattr( new_group.grouphash_set.order_by('-id').first(), 'id', None), 'old_hash_id': getattr( group.grouphash_set.order_by('-id').first(), 'id', None), }) model_list = ( Activity, GroupAssignee, GroupHash, GroupRuleStatus, GroupSubscription, GroupTagValue, GroupTagKey, EventMapping, Event, UserReport, GroupRedirect, GroupMeta, ) has_more = merge_objects( model_list, group, new_group, logger=logger, transaction_id=transaction_id, ) if has_more: merge_group.delay( from_object_id=from_object_id, to_object_id=to_object_id, transaction_id=transaction_id, recursed=True, ) return features.merge(new_group, [group], allow_unsafe=True) for model in [tsdb.models.group]: tsdb.merge(model, new_group.id, [group.id]) for model in [tsdb.models.users_affected_by_group]: tsdb.merge_distinct_counts(model, new_group.id, [group.id]) for model in [ tsdb.models.frequent_releases_by_group, tsdb.models.frequent_environments_by_group ]: tsdb.merge_frequencies(model, new_group.id, [group.id]) previous_group_id = group.id group.delete() delete_logger.info('object.delete.executed', extra={ 'object_id': previous_group_id, 'transaction_id': transaction_id, 'model': Group.__name__, }) try: with transaction.atomic(): GroupRedirect.objects.create( group_id=new_group.id, previous_group_id=previous_group_id, ) except IntegrityError: pass new_group.update( # TODO(dcramer): ideally these would be SQL clauses first_seen=min(group.first_seen, new_group.first_seen), last_seen=max(group.last_seen, new_group.last_seen), ) try: # it's possible to hit an out of range value for counters new_group.update( times_seen=F('times_seen') + group.times_seen, num_comments=F('num_comments') + group.num_comments, ) except DataError: pass
def merge_groups(from_object_ids=None, to_object_id=None, transaction_id=None, recursed=False, eventstream_state=None, **kwargs): # TODO(mattrobenolt): Write tests for all of this from sentry.models import ( Activity, Group, GroupAssignee, GroupEnvironment, GroupHash, GroupRuleStatus, GroupSubscription, Environment, EventAttachment, UserReport, GroupRedirect, GroupMeta, get_group_with_redirect, ) if not (from_object_ids and to_object_id): logger.error("group.malformed.missing_params", extra={"transaction_id": transaction_id}) return False # Operate on one "from" group per task iteration. The task is recursed # until each group has been merged. from_object_id = from_object_ids[0] try: new_group, _ = get_group_with_redirect(to_object_id) except Group.DoesNotExist: logger.warn( "group.malformed.invalid_id", extra={ "transaction_id": transaction_id, "old_object_ids": from_object_ids }, ) return False if not recursed: logger.info( "merge.queued", extra={ "transaction_id": transaction_id, "new_group_id": new_group.id, "old_group_ids": from_object_ids, # TODO(jtcunning): figure out why these are full seq scans and/or alternative solution # 'new_event_id': getattr(new_group.event_set.order_by('-id').first(), 'id', None), # 'old_event_id': getattr(group.event_set.order_by('-id').first(), 'id', None), # 'new_hash_id': getattr(new_group.grouphash_set.order_by('-id').first(), 'id', None), # 'old_hash_id': getattr(group.grouphash_set.order_by('-id').first(), 'id', None), }, ) try: group = Group.objects.select_related("project").get(id=from_object_id) except Group.DoesNotExist: from_object_ids.remove(from_object_id) logger.warn( "group.malformed.invalid_id", extra={ "transaction_id": transaction_id, "old_object_id": from_object_id }, ) else: model_list = tuple(EXTRA_MERGE_MODELS) + ( Activity, GroupAssignee, GroupEnvironment, GroupHash, GroupRuleStatus, GroupSubscription, EventAttachment, UserReport, GroupRedirect, GroupMeta, ) has_more = merge_objects(model_list, group, new_group, logger=logger, transaction_id=transaction_id) if not has_more: # There are no more objects to merge for *this* "from" group, remove it # from the list of "from" groups that are being merged, and finish the # work for this group. from_object_ids.remove(from_object_id) similarity.merge(group.project, new_group, [group], allow_unsafe=True) environment_ids = list( Environment.objects.filter(projects=group.project).values_list( "id", flat=True)) for model in [tsdb.models.group]: tsdb.merge( model, new_group.id, [group.id], environment_ids=environment_ids if model in tsdb.models_with_environment_support else None, ) for model in [tsdb.models.users_affected_by_group]: tsdb.merge_distinct_counts( model, new_group.id, [group.id], environment_ids=environment_ids if model in tsdb.models_with_environment_support else None, ) for model in [ tsdb.models.frequent_releases_by_group, tsdb.models.frequent_environments_by_group, ]: tsdb.merge_frequencies( model, new_group.id, [group.id], environment_ids=environment_ids if model in tsdb.models_with_environment_support else None, ) previous_group_id = group.id with transaction.atomic(): GroupRedirect.create_for_group(group, new_group) group.delete() delete_logger.info( "object.delete.executed", extra={ "object_id": previous_group_id, "transaction_id": transaction_id, "model": Group.__name__, }, ) new_group.update( # TODO(dcramer): ideally these would be SQL clauses first_seen=min(group.first_seen, new_group.first_seen), last_seen=max(group.last_seen, new_group.last_seen), ) try: # it's possible to hit an out of range value for counters new_group.update( times_seen=F("times_seen") + group.times_seen, num_comments=F("num_comments") + group.num_comments, ) except DataError: pass if from_object_ids: # This task is recursed until `from_object_ids` is empty and all # "from" groups have merged into the `to_group_id`. merge_groups.delay( from_object_ids=from_object_ids, to_object_id=to_object_id, transaction_id=transaction_id, recursed=True, eventstream_state=eventstream_state, ) elif eventstream_state: # All `from_object_ids` have been merged! eventstream.end_merge(eventstream_state)
def merge_group( from_object_id=None, to_object_id=None, transaction_id=None, recursed=False, eventstream_state=None, **kwargs ): # TODO(mattrobenolt): Write tests for all of this from sentry.models import ( Activity, Group, GroupAssignee, GroupEnvironment, GroupHash, GroupRuleStatus, GroupSubscription, Environment, EventMapping, Event, UserReport, GroupRedirect, GroupMeta, ) if not (from_object_id and to_object_id): logger.error( 'group.malformed.missing_params', extra={ 'transaction_id': transaction_id, } ) return try: group = Group.objects.get(id=from_object_id) except Group.DoesNotExist: logger.warn( 'group.malformed.invalid_id', extra={ 'transaction_id': transaction_id, 'old_object_id': from_object_id, } ) return try: new_group = Group.objects.get(id=to_object_id) except Group.DoesNotExist: logger.warn( 'group.malformed.invalid_id', extra={ 'transaction_id': transaction_id, 'old_object_id': from_object_id, } ) return if not recursed: logger.info( 'merge.queued', extra={ 'transaction_id': transaction_id, 'new_group_id': new_group.id, 'old_group_id': group.id, # TODO(jtcunning): figure out why these are full seq scans and/or alternative solution # 'new_event_id': getattr(new_group.event_set.order_by('-id').first(), 'id', None), # 'old_event_id': getattr(group.event_set.order_by('-id').first(), 'id', None), # 'new_hash_id': getattr(new_group.grouphash_set.order_by('-id').first(), 'id', None), # 'old_hash_id': getattr(group.grouphash_set.order_by('-id').first(), 'id', None), } ) model_list = tuple(EXTRA_MERGE_MODELS) + ( Activity, GroupAssignee, GroupEnvironment, GroupHash, GroupRuleStatus, GroupSubscription, EventMapping, Event, UserReport, GroupRedirect, GroupMeta, ) has_more = merge_objects( model_list, group, new_group, logger=logger, transaction_id=transaction_id, ) last_task = False if has_more: merge_group.delay( from_object_id=from_object_id, to_object_id=to_object_id, transaction_id=transaction_id, recursed=True, eventstream_state=eventstream_state, ) return else: last_task = True features.merge(new_group, [group], allow_unsafe=True) environment_ids = list( Environment.objects.filter( projects=group.project ).values_list('id', flat=True) ) for model in [tsdb.models.group]: tsdb.merge( model, new_group.id, [group.id], environment_ids=environment_ids if model in tsdb.models_with_environment_support else None ) for model in [tsdb.models.users_affected_by_group]: tsdb.merge_distinct_counts( model, new_group.id, [group.id], environment_ids=environment_ids if model in tsdb.models_with_environment_support else None, ) for model in [ tsdb.models.frequent_releases_by_group, tsdb.models.frequent_environments_by_group ]: tsdb.merge_frequencies( model, new_group.id, [group.id], environment_ids=environment_ids if model in tsdb.models_with_environment_support else None, ) previous_group_id = group.id group.delete() delete_logger.info( 'object.delete.executed', extra={ 'object_id': previous_group_id, 'transaction_id': transaction_id, 'model': Group.__name__, } ) try: with transaction.atomic(): GroupRedirect.objects.create( group_id=new_group.id, previous_group_id=previous_group_id, ) except IntegrityError: pass new_group.update( # TODO(dcramer): ideally these would be SQL clauses first_seen=min(group.first_seen, new_group.first_seen), last_seen=max(group.last_seen, new_group.last_seen), ) try: # it's possible to hit an out of range value for counters new_group.update( times_seen=F('times_seen') + group.times_seen, num_comments=F('num_comments') + group.num_comments, ) except DataError: pass if last_task and eventstream_state: eventstream.end_merge(eventstream_state)
def merge_group( from_object_id=None, to_object_id=None, transaction_id=None, recursed=False, **kwargs ): # TODO(mattrobenolt): Write tests for all of this from sentry.models import ( Activity, Group, GroupAssignee, GroupHash, GroupRuleStatus, GroupSubscription, EventMapping, Event, UserReport, GroupRedirect, GroupMeta, ) if not (from_object_id and to_object_id): logger.error( 'group.malformed.missing_params', extra={ 'transaction_id': transaction_id, } ) return try: group = Group.objects.get(id=from_object_id) except Group.DoesNotExist: logger.warn( 'group.malformed.invalid_id', extra={ 'transaction_id': transaction_id, 'old_object_id': from_object_id, } ) return try: new_group = Group.objects.get(id=to_object_id) except Group.DoesNotExist: logger.warn( 'group.malformed.invalid_id', extra={ 'transaction_id': transaction_id, 'old_object_id': from_object_id, } ) return if not recursed: logger.info( 'merge.queued', extra={ 'transaction_id': transaction_id, 'new_group_id': new_group.id, 'old_group_id': group.id, # TODO(jtcunning): figure out why these are full seq scans and/or alternative solution # 'new_event_id': getattr(new_group.event_set.order_by('-id').first(), 'id', None), # 'old_event_id': getattr(group.event_set.order_by('-id').first(), 'id', None), # 'new_hash_id': getattr(new_group.grouphash_set.order_by('-id').first(), 'id', None), # 'old_hash_id': getattr(group.grouphash_set.order_by('-id').first(), 'id', None), } ) model_list = tuple(EXTRA_MERGE_MODELS) + ( Activity, GroupAssignee, GroupHash, GroupRuleStatus, GroupSubscription, EventMapping, Event, UserReport, GroupRedirect, GroupMeta, ) has_more = merge_objects( model_list, group, new_group, logger=logger, transaction_id=transaction_id, ) if has_more: merge_group.delay( from_object_id=from_object_id, to_object_id=to_object_id, transaction_id=transaction_id, recursed=True, ) return features.merge(new_group, [group], allow_unsafe=True) for model in [tsdb.models.group]: tsdb.merge(model, new_group.id, [group.id]) for model in [tsdb.models.users_affected_by_group]: tsdb.merge_distinct_counts(model, new_group.id, [group.id]) for model in [ tsdb.models.frequent_releases_by_group, tsdb.models.frequent_environments_by_group ]: tsdb.merge_frequencies(model, new_group.id, [group.id]) previous_group_id = group.id group.delete() delete_logger.info( 'object.delete.executed', extra={ 'object_id': previous_group_id, 'transaction_id': transaction_id, 'model': Group.__name__, } ) try: with transaction.atomic(): GroupRedirect.objects.create( group_id=new_group.id, previous_group_id=previous_group_id, ) except IntegrityError: pass new_group.update( # TODO(dcramer): ideally these would be SQL clauses first_seen=min(group.first_seen, new_group.first_seen), last_seen=max(group.last_seen, new_group.last_seen), ) try: # it's possible to hit an out of range value for counters new_group.update( times_seen=F('times_seen') + group.times_seen, num_comments=F('num_comments') + group.num_comments, ) except DataError: pass
def merge_groups( from_object_ids=None, to_object_id=None, transaction_id=None, recursed=False, eventstream_state=None, **kwargs ): # TODO(mattrobenolt): Write tests for all of this from sentry.models import ( Activity, Group, GroupAssignee, GroupEnvironment, GroupHash, GroupRuleStatus, GroupSubscription, Environment, EventMapping, Event, UserReport, GroupRedirect, GroupMeta, get_group_with_redirect, ) if not (from_object_ids and to_object_id): logger.error( 'group.malformed.missing_params', extra={ 'transaction_id': transaction_id, } ) return # Operate on one "from" group per task iteration. The task is recursed # until each group has been merged. from_object_id = from_object_ids[0] try: new_group, _ = get_group_with_redirect(to_object_id) except Group.DoesNotExist: logger.warn( 'group.malformed.invalid_id', extra={ 'transaction_id': transaction_id, 'old_object_ids': from_object_ids, } ) return if not recursed: logger.info( 'merge.queued', extra={ 'transaction_id': transaction_id, 'new_group_id': new_group.id, 'old_group_ids': from_object_ids, # TODO(jtcunning): figure out why these are full seq scans and/or alternative solution # 'new_event_id': getattr(new_group.event_set.order_by('-id').first(), 'id', None), # 'old_event_id': getattr(group.event_set.order_by('-id').first(), 'id', None), # 'new_hash_id': getattr(new_group.grouphash_set.order_by('-id').first(), 'id', None), # 'old_hash_id': getattr(group.grouphash_set.order_by('-id').first(), 'id', None), } ) try: group = Group.objects.get(id=from_object_id) except Group.DoesNotExist: from_object_ids.remove(from_object_id) logger.warn( 'group.malformed.invalid_id', extra={ 'transaction_id': transaction_id, 'old_object_id': from_object_id, } ) else: model_list = tuple(EXTRA_MERGE_MODELS) + ( Activity, GroupAssignee, GroupEnvironment, GroupHash, GroupRuleStatus, GroupSubscription, EventMapping, Event, UserReport, GroupRedirect, GroupMeta, ) has_more = merge_objects( model_list, group, new_group, logger=logger, transaction_id=transaction_id, ) if not has_more: # There are no more objects to merge for *this* "from" group, remove it # from the list of "from" groups that are being merged, and finish the # work for this group. from_object_ids.remove(from_object_id) features.merge(new_group, [group], allow_unsafe=True) environment_ids = list( Environment.objects.filter( projects=group.project ).values_list('id', flat=True) ) for model in [tsdb.models.group]: tsdb.merge( model, new_group.id, [group.id], environment_ids=environment_ids if model in tsdb.models_with_environment_support else None ) for model in [tsdb.models.users_affected_by_group]: tsdb.merge_distinct_counts( model, new_group.id, [group.id], environment_ids=environment_ids if model in tsdb.models_with_environment_support else None, ) for model in [ tsdb.models.frequent_releases_by_group, tsdb.models.frequent_environments_by_group ]: tsdb.merge_frequencies( model, new_group.id, [group.id], environment_ids=environment_ids if model in tsdb.models_with_environment_support else None, ) previous_group_id = group.id group.delete() delete_logger.info( 'object.delete.executed', extra={ 'object_id': previous_group_id, 'transaction_id': transaction_id, 'model': Group.__name__, } ) try: with transaction.atomic(): GroupRedirect.objects.create( group_id=new_group.id, previous_group_id=previous_group_id, ) except IntegrityError: pass new_group.update( # TODO(dcramer): ideally these would be SQL clauses first_seen=min(group.first_seen, new_group.first_seen), last_seen=max(group.last_seen, new_group.last_seen), ) try: # it's possible to hit an out of range value for counters new_group.update( times_seen=F('times_seen') + group.times_seen, num_comments=F('num_comments') + group.num_comments, ) except DataError: pass if from_object_ids: # This task is recursed until `from_object_ids` is empty and all # "from" groups have merged into the `to_group_id`. merge_groups.delay( from_object_ids=from_object_ids, to_object_id=to_object_id, transaction_id=transaction_id, recursed=True, eventstream_state=eventstream_state, ) return # All `from_object_ids` have been merged! if eventstream_state: eventstream.end_merge(eventstream_state)