def get_attrs(self, item_list, user): from sentry.integrations import IntegrationFeatures from sentry.models import PlatformExternalIssue from sentry.plugins.base import plugins GroupMeta.objects.populate_cache(item_list) # Note that organization is necessary here for use in `_get_permalink` to avoid # making unnecessary queries. attach_foreignkey(item_list, Group.project, related=("organization", )) if user.is_authenticated and item_list: bookmarks = set( GroupBookmark.objects.filter( user=user, group__in=item_list).values_list("group_id", flat=True)) seen_groups = dict( GroupSeen.objects.filter(user=user, group__in=item_list).values_list( "group_id", "last_seen")) subscriptions = self._get_subscriptions(item_list, user) else: bookmarks = set() seen_groups = {} subscriptions = defaultdict(lambda: (False, False, None)) assignees = { a.group_id: a.assigned_actor() for a in GroupAssignee.objects.filter(group__in=item_list) } resolved_assignees = ActorTuple.resolve_dict(assignees) ignore_items = { g.group_id: g for g in GroupSnooze.objects.filter(group__in=item_list) } resolved_item_list = [ i for i in item_list if i.status == GroupStatus.RESOLVED ] if resolved_item_list: release_resolutions = { i[0]: i[1:] for i in GroupResolution.objects.filter( group__in=resolved_item_list).values_list( "group", "type", "release__version", "actor_id") } # due to our laziness, and django's inability to do a reasonable join here # we end up with two queries commit_results = list( Commit.objects.extra( select={"group_id": "sentry_grouplink.group_id"}, tables=["sentry_grouplink"], where=[ "sentry_grouplink.linked_id = sentry_commit.id", "sentry_grouplink.group_id IN ({})".format(", ".join( str(i.id) for i in resolved_item_list)), "sentry_grouplink.linked_type = %s", "sentry_grouplink.relationship = %s", ], params=[ int(GroupLink.LinkedType.commit), int(GroupLink.Relationship.resolves) ], )) commit_resolutions = { i.group_id: d for i, d in zip(commit_results, serialize( commit_results, user)) } else: release_resolutions = {} commit_resolutions = {} actor_ids = {r[-1] for r in release_resolutions.values()} actor_ids.update(r.actor_id for r in ignore_items.values()) if actor_ids: users = list(User.objects.filter(id__in=actor_ids, is_active=True)) actors = {u.id: d for u, d in zip(users, serialize(users, user))} else: actors = {} share_ids = dict( GroupShare.objects.filter(group__in=item_list).values_list( "group_id", "uuid")) result = {} seen_stats = self._get_seen_stats(item_list, user) annotations_by_group_id = defaultdict(list) organization_id_list = list( {item.project.organization_id for item in item_list}) # if no groups, then we can't proceed but this seems to be a valid use case if not item_list: return {} if len(organization_id_list) > 1: # this should never happen but if it does we should know about it logger.warning( "Found multiple organizations for groups: %s, with orgs: %s" % ([item.id for item in item_list], organization_id_list)) # should only have 1 org at this point organization_id = organization_id_list[0] # find all the integration installs that have issue tracking for integration in Integration.objects.filter( organizations=organization_id): if not (integration.has_feature(IntegrationFeatures.ISSUE_BASIC) or integration.has_feature(IntegrationFeatures.ISSUE_SYNC)): continue install = integration.get_installation(organization_id) local_annotations_by_group_id = (safe_execute( install.get_annotations_for_group_list, group_list=item_list, _with_transaction=False, ) or {}) merge_list_dictionaries(annotations_by_group_id, local_annotations_by_group_id) # find the external issues for sentry apps and add them in local_annotations_by_group_id = (safe_execute( PlatformExternalIssue.get_annotations_for_group_list, group_list=item_list, _with_transaction=False, ) or {}) merge_list_dictionaries(annotations_by_group_id, local_annotations_by_group_id) snuba_stats = self._get_group_snuba_stats(item_list, seen_stats) for item in item_list: active_date = item.active_at or item.first_seen annotations = [] annotations.extend(annotations_by_group_id[item.id]) # add the annotations for plugins # note that the model GroupMeta where all the information is stored is already cached at the top of this function # so these for loops doesn't make a bunch of queries for plugin in plugins.for_project(project=item.project, version=1): safe_execute(plugin.tags, None, item, annotations, _with_transaction=False) for plugin in plugins.for_project(project=item.project, version=2): annotations.extend( safe_execute(plugin.get_annotations, group=item, _with_transaction=False) or ()) resolution_actor = None resolution_type = None resolution = release_resolutions.get(item.id) if resolution: resolution_type = "release" resolution_actor = actors.get(resolution[-1]) if not resolution: resolution = commit_resolutions.get(item.id) if resolution: resolution_type = "commit" ignore_item = ignore_items.get(item.id) if ignore_item: ignore_actor = actors.get(ignore_item.actor_id) else: ignore_actor = None result[item] = { "id": item.id, "assigned_to": resolved_assignees.get(item.id), "is_bookmarked": item.id in bookmarks, "subscription": subscriptions[item.id], "has_seen": seen_groups.get(item.id, active_date) > active_date, "annotations": annotations, "ignore_until": ignore_item, "ignore_actor": ignore_actor, "resolution": resolution, "resolution_type": resolution_type, "resolution_actor": resolution_actor, "share_id": share_ids.get(item.id), } result[item]["is_unhandled"] = bool( snuba_stats.get(item.id, {}).get("unhandled")) if seen_stats: result[item].update(seen_stats.get(item, {})) return result
def test_call(self): plugin = plugins.get("example") plugin.enable(self.project) self.migrator.call() assert plugin not in plugins.for_project(self.project)
def test_does_not_disable_any_plugin(self): plugin = plugins.get("webhooks") plugin.enable(self.project) self.migrator.call() assert plugin in plugins.for_project(self.project)
def save(self, project_id, raw=False, assume_normalized=False): """ We re-insert events with duplicate IDs into Snuba, which is responsible for deduplicating events. Since deduplication in Snuba is on the primary key (based on event ID, project ID and day), events with same IDs are only deduplicated if their timestamps fall on the same day. The latest event always wins and overwrites the value of events received earlier in that day. Since we increment counters and frequencies here before events get inserted to eventstream these numbers may be larger than the total number of events if we receive duplicate event IDs that fall on the same day (that do not hit cache first). """ # Normalize if needed if not self._normalized: if not assume_normalized: self.normalize() self._normalized = True data = self._data project = Project.objects.get_from_cache(id=project_id) project._organization_cache = Organization.objects.get_from_cache( id=project.organization_id) # Pull out the culprit culprit = self.get_culprit() # Pull the toplevel data we're interested in level = data.get("level") # TODO(mitsuhiko): this code path should be gone by July 2018. # This is going to be fine because no code actually still depends # on integers here. When we need an integer it will be converted # into one later. Old workers used to send integers here. if level is not None and isinstance(level, six.integer_types): level = LOG_LEVELS[level] transaction_name = data.get("transaction") logger_name = data.get("logger") release = data.get("release") dist = data.get("dist") environment = data.get("environment") recorded_timestamp = data.get("timestamp") # We need to swap out the data with the one internal to the newly # created event object event = self._get_event_instance(project_id=project_id) self._data = data = event.data.data event._project_cache = project date = event.datetime platform = event.platform event_id = event.event_id if transaction_name: transaction_name = force_text(transaction_name) # Right now the event type is the signal to skip the group. This # is going to change a lot. if event.get_event_type() == "transaction": issueless_event = True else: issueless_event = False # Some of the data that are toplevel attributes are duplicated # into tags (logger, level, environment, transaction). These are # different from legacy attributes which are normalized into tags # ahead of time (site, server_name). setdefault_path(data, "tags", value=[]) set_tag(data, "level", level) if logger_name: set_tag(data, "logger", logger_name) if environment: set_tag(data, "environment", environment) if transaction_name: set_tag(data, "transaction", transaction_name) if release: # dont allow a conflicting 'release' tag pop_tag(data, "release") release = Release.get_or_create(project=project, version=release, date_added=date) set_tag(data, "sentry:release", release.version) if dist and release: dist = release.add_dist(dist, date) # dont allow a conflicting 'dist' tag pop_tag(data, "dist") set_tag(data, "sentry:dist", dist.name) else: dist = None event_user = self._get_event_user(project, data) if event_user: # dont allow a conflicting 'user' tag pop_tag(data, "user") set_tag(data, "sentry:user", event_user.tag_value) # At this point we want to normalize the in_app values in case the # clients did not set this appropriately so far. grouping_config = load_grouping_config( get_grouping_config_dict_for_event_data(data, project)) normalize_stacktraces_for_grouping(data, grouping_config) for plugin in plugins.for_project(project, version=None): added_tags = safe_execute(plugin.get_tags, event, _with_transaction=False) if added_tags: # plugins should not override user provided tags for key, value in added_tags: if get_tag(data, key) is None: set_tag(data, key, value) for path, iface in six.iteritems(event.interfaces): for k, v in iface.iter_tags(): set_tag(data, k, v) # Get rid of ephemeral interface data if iface.ephemeral: data.pop(iface.path, None) # The active grouping config was put into the event in the # normalize step before. We now also make sure that the # fingerprint was set to `'{{ default }}' just in case someone # removed it from the payload. The call to get_hashes will then # look at `grouping_config` to pick the right parameters. data["fingerprint"] = data.get("fingerprint") or ["{{ default }}"] apply_server_fingerprinting( data, get_fingerprinting_config_for_project(project)) # Here we try to use the grouping config that was requested in the # event. If that config has since been deleted (because it was an # experimental grouping config) we fall back to the default. try: hashes = event.get_hashes() except GroupingConfigNotFound: data["grouping_config"] = get_grouping_config_dict_for_project( project) hashes = event.get_hashes() data["hashes"] = hashes # we want to freeze not just the metadata and type in but also the # derived attributes. The reason for this is that we push this # data into kafka for snuba processing and our postprocessing # picks up the data right from the snuba topic. For most usage # however the data is dynamically overridden by Event.title and # Event.location (See Event.as_dict) materialized_metadata = self.materialize_metadata() data.update(materialized_metadata) data["culprit"] = culprit received_timestamp = event.data.get("received") or float( event.datetime.strftime("%s")) if not issueless_event: # The group gets the same metadata as the event when it's flushed but # additionally the `last_received` key is set. This key is used by # _save_aggregate. group_metadata = dict(materialized_metadata) group_metadata["last_received"] = received_timestamp kwargs = { "platform": platform, "message": event.search_message, "culprit": culprit, "logger": logger_name, "level": LOG_LEVELS_MAP.get(level), "last_seen": date, "first_seen": date, "active_at": date, "data": group_metadata, } if release: kwargs["first_release"] = release try: group, is_new, is_regression = self._save_aggregate( event=event, hashes=hashes, release=release, **kwargs) except HashDiscarded: event_discarded.send_robust(project=project, sender=EventManager) metrics.incr( "events.discarded", skip_internal=True, tags={ "organization_id": project.organization_id, "platform": platform }, ) raise else: event_saved.send_robust(project=project, event_size=event.size, sender=EventManager) event.group = group else: group = None is_new = False is_regression = False event_saved.send_robust(project=project, event_size=event.size, sender=EventManager) # store a reference to the group id to guarantee validation of isolation event.data.bind_ref(event) environment = Environment.get_or_create(project=project, name=environment) if group: group_environment, is_new_group_environment = GroupEnvironment.get_or_create( group_id=group.id, environment_id=environment.id, defaults={"first_release": release if release else None}, ) else: is_new_group_environment = False if release: ReleaseEnvironment.get_or_create(project=project, release=release, environment=environment, datetime=date) ReleaseProjectEnvironment.get_or_create(project=project, release=release, environment=environment, datetime=date) if group: grouprelease = GroupRelease.get_or_create( group=group, release=release, environment=environment, datetime=date) counters = [(tsdb.models.project, project.id)] if group: counters.append((tsdb.models.group, group.id)) if release: counters.append((tsdb.models.release, release.id)) tsdb.incr_multi(counters, timestamp=event.datetime, environment_id=environment.id) frequencies = [] if group: frequencies.append((tsdb.models.frequent_environments_by_group, { group.id: { environment.id: 1 } })) if release: frequencies.append((tsdb.models.frequent_releases_by_group, { group.id: { grouprelease.id: 1 } })) if frequencies: tsdb.record_frequency_multi(frequencies, timestamp=event.datetime) if group: UserReport.objects.filter(project=project, event_id=event_id).update( group=group, environment=environment) # Write the event to Nodestore event.data.save() if event_user: counters = [(tsdb.models.users_affected_by_project, project.id, (event_user.tag_value, ))] if group: counters.append((tsdb.models.users_affected_by_group, group.id, (event_user.tag_value, ))) tsdb.record_multi(counters, timestamp=event.datetime, environment_id=environment.id) if release: if is_new: buffer.incr( ReleaseProject, {"new_groups": 1}, { "release_id": release.id, "project_id": project.id }, ) if is_new_group_environment: buffer.incr( ReleaseProjectEnvironment, {"new_issues_count": 1}, { "project_id": project.id, "release_id": release.id, "environment_id": environment.id, }, ) if not raw: if not project.first_event: project.update(first_event=date) first_event_received.send_robust(project=project, event=event, sender=Project) eventstream.insert( group=group, event=event, is_new=is_new, is_regression=is_regression, is_new_group_environment=is_new_group_environment, primary_hash=hashes[0], # We are choosing to skip consuming the event back # in the eventstream if it's flagged as raw. # This means that we want to publish the event # through the event stream, but we don't care # about post processing and handling the commit. skip_consume=raw, ) metric_tags = {"from_relay": "_relay_processed" in self._data} metrics.timing("events.latency", received_timestamp - recorded_timestamp, tags=metric_tags) metrics.timing("events.size.data.post_save", event.size, tags=metric_tags) metrics.incr( "events.post_save.normalize.errors", amount=len(self._data.get("errors") or ()), tags=metric_tags, ) return event
def notify(self, notification, target_type, target_identifier=None, **kwargs): metrics.incr("mail_adapter.notify") event = notification.event environment = event.get_tag("environment") group = event.group project = group.project org = group.organization logger.info( "mail.adapter.notify", extra={ "target_type": target_type.value, "target_identifier": target_identifier, "group": group.id, "project_id": project.id, }, ) subject = event.get_email_subject() query_params = {"referrer": "alert_email"} if environment: query_params["environment"] = environment link = group.get_absolute_url(params=query_params) template = "sentry/emails/error.txt" html_template = "sentry/emails/error.html" rules = [] for rule in notification.rules: rule_link = f"/organizations/{org.slug}/alerts/rules/{project.slug}/{rule.id}/" rules.append((rule.label, rule_link)) enhanced_privacy = org.flags.enhanced_privacy # lets identify possibly suspect commits and owners commits = {} try: committers = get_serialized_event_file_committers(project, event) except (Commit.DoesNotExist, Release.DoesNotExist): pass except Exception as exc: logging.exception(str(exc)) else: for committer in committers: for commit in committer["commits"]: if commit["id"] not in commits: commit_data = commit.copy() commit_data["shortId"] = commit_data["id"][:7] commit_data["author"] = committer["author"] commit_data["subject"] = commit_data["message"].split( "\n", 1)[0] commits[commit["id"]] = commit_data project_plugins = plugins.for_project(project, version=1) organization_integrations = Integration.objects.filter( organizations=org).first() has_integrations = bool(project_plugins or organization_integrations) context = { "subject": subject, "template": template, "html_template": html_template, "project": project, "project_label": project.get_full_name(), "group": group, "event": event, "link": link, "rules": rules, "has_integrations": has_integrations, "enhanced_privacy": enhanced_privacy, "commits": sorted(commits.values(), key=lambda x: x["score"], reverse=True), "environment": environment, } # if the organization has enabled enhanced privacy controls we dont send # data which may show PII or source code if not enhanced_privacy: interface_list = [] for interface in event.interfaces.values(): body = interface.to_email_html(event) if not body: continue text_body = interface.to_string(event) interface_list.append( (interface.get_title(), mark_safe(body), text_body)) context.update({"tags": event.tags, "interfaces": interface_list}) context["headers"] = { "X-Sentry-Logger": group.logger, "X-Sentry-Logger-Level": group.get_level_display(), "X-Sentry-Project": project.slug, "X-Sentry-Reply-To": group_id_to_email(group.id), "X-SMTPAPI": json.dumps({"category": "issue_alert_email"}), } context["target_type"] = target_type context["target_identifier"] = target_identifier participants_by_provider = self.get_send_to( project=project, target_type=target_type, target_identifier=target_identifier, event=event, ) for provider, participants in participants_by_provider.items(): notify_participants(self, provider, participants, context)
def get_attrs(self, item_list, user): from sentry.plugins.base import plugins GroupMeta.objects.populate_cache(item_list) attach_foreignkey(item_list, Group.project) if user.is_authenticated() and item_list: bookmarks = set( GroupBookmark.objects.filter( user=user, group__in=item_list).values_list("group_id", flat=True)) seen_groups = dict( GroupSeen.objects.filter(user=user, group__in=item_list).values_list( "group_id", "last_seen")) subscriptions = self._get_subscriptions(item_list, user) else: bookmarks = set() seen_groups = {} subscriptions = defaultdict(lambda: (False, None)) assignees = { a.group_id: a.assigned_actor() for a in GroupAssignee.objects.filter(group__in=item_list) } resolved_assignees = Actor.resolve_dict(assignees) ignore_items = { g.group_id: g for g in GroupSnooze.objects.filter(group__in=item_list) } resolved_item_list = [ i for i in item_list if i.status == GroupStatus.RESOLVED ] if resolved_item_list: release_resolutions = { i[0]: i[1:] for i in GroupResolution.objects.filter( group__in=resolved_item_list).values_list( "group", "type", "release__version", "actor_id") } # due to our laziness, and django's inability to do a reasonable join here # we end up with two queries commit_results = list( Commit.objects.extra( select={"group_id": "sentry_grouplink.group_id"}, tables=["sentry_grouplink"], where=[ "sentry_grouplink.linked_id = sentry_commit.id", "sentry_grouplink.group_id IN ({})".format(", ".join( six.text_type(i.id) for i in resolved_item_list)), "sentry_grouplink.linked_type = %s", "sentry_grouplink.relationship = %s", ], params=[ int(GroupLink.LinkedType.commit), int(GroupLink.Relationship.resolves) ], )) commit_resolutions = { i.group_id: d for i, d in itertools.izip(commit_results, serialize(commit_results, user)) } else: release_resolutions = {} commit_resolutions = {} actor_ids = set(r[-1] for r in six.itervalues(release_resolutions)) actor_ids.update(r.actor_id for r in six.itervalues(ignore_items)) if actor_ids: users = list(User.objects.filter(id__in=actor_ids, is_active=True)) actors = { u.id: d for u, d in itertools.izip(users, serialize(users, user)) } else: actors = {} share_ids = dict( GroupShare.objects.filter(group__in=item_list).values_list( "group_id", "uuid")) result = {} seen_stats = self._get_seen_stats(item_list, user) for item in item_list: active_date = item.active_at or item.first_seen annotations = [] for plugin in plugins.for_project(project=item.project, version=1): safe_execute(plugin.tags, None, item, annotations, _with_transaction=False) for plugin in plugins.for_project(project=item.project, version=2): annotations.extend( safe_execute(plugin.get_annotations, group=item, _with_transaction=False) or ()) from sentry.integrations import IntegrationFeatures for integration in Integration.objects.filter( organizations=item.project.organization_id): if not (integration.has_feature( IntegrationFeatures.ISSUE_BASIC) or integration.has_feature( IntegrationFeatures.ISSUE_SYNC)): continue install = integration.get_installation( item.project.organization_id) annotations.extend( safe_execute(install.get_annotations, group=item, _with_transaction=False) or ()) from sentry.models import PlatformExternalIssue annotations.extend( safe_execute(PlatformExternalIssue.get_annotations, group=item, _with_transaction=False) or ()) resolution_actor = None resolution_type = None resolution = release_resolutions.get(item.id) if resolution: resolution_type = "release" resolution_actor = actors.get(resolution[-1]) if not resolution: resolution = commit_resolutions.get(item.id) if resolution: resolution_type = "commit" ignore_item = ignore_items.get(item.id) if ignore_item: ignore_actor = actors.get(ignore_item.actor_id) else: ignore_actor = None result[item] = { "assigned_to": resolved_assignees.get(item.id), "is_bookmarked": item.id in bookmarks, "subscription": subscriptions[item.id], "has_seen": seen_groups.get(item.id, active_date) > active_date, "annotations": annotations, "ignore_until": ignore_item, "ignore_actor": ignore_actor, "resolution": resolution, "resolution_type": resolution_type, "resolution_actor": resolution_actor, "share_id": share_ids.get(item.id), } result[item].update(seen_stats.get(item, {})) return result
def post_process_group( is_new, is_regression, is_new_group_environment, cache_key, group_id=None, **kwargs ): """ Fires post processing hooks for a group. """ from sentry.eventstore.models import Event from sentry.eventstore.processing import event_processing_store from sentry.utils import snuba from sentry.reprocessing2 import is_reprocessed_event with snuba.options_override({"consistent": True}): # We use the data being present/missing in the processing store # to ensure that we don't duplicate work should the forwarding consumers # need to rewind history. data = event_processing_store.get(cache_key) if not data: logger.info( "post_process.skipped", extra={"cache_key": cache_key, "reason": "missing_cache"}, ) return event = Event( project_id=data["project"], event_id=data["event_id"], group_id=group_id, data=data ) set_current_project(event.project_id) is_reprocessed = is_reprocessed_event(event.data) # NOTE: we must pass through the full Event object, and not an # event_id since the Event object may not actually have been stored # in the database due to sampling. from sentry.models import ( Commit, Project, Organization, EventDict, GroupInboxReason, ) from sentry.models.groupinbox import add_group_to_inbox from sentry.models.group import get_group_with_redirect from sentry.rules.processor import RuleProcessor from sentry.tasks.servicehooks import process_service_hook from sentry.tasks.groupowner import process_suspect_commits # Re-bind node data to avoid renormalization. We only want to # renormalize when loading old data from the database. event.data = EventDict(event.data, skip_renormalization=True) # Re-bind Project and Org since we're reading the Event object # from cache which may contain stale parent models. event.project = Project.objects.get_from_cache(id=event.project_id) event.project._organization_cache = Organization.objects.get_from_cache( id=event.project.organization_id ) if event.group_id: # Re-bind Group since we're reading the Event object # from cache, which may contain a stale group and project event.group, _ = get_group_with_redirect(event.group_id) event.group_id = event.group.id event.group.project = event.project event.group.project._organization_cache = event.project._organization_cache bind_organization_context(event.project.organization) _capture_stats(event, is_new) if event.group_id and not is_reprocessed: # we process snoozes before rules as it might create a regression # but not if it's new because you can't immediately snooze a new group has_reappeared = False if is_new else process_snoozes(event.group) if not has_reappeared: # If true, we added the .UNIGNORED reason already if is_new: add_group_to_inbox(event.group, GroupInboxReason.NEW) elif is_regression: add_group_to_inbox(event.group, GroupInboxReason.REGRESSION) handle_owner_assignment(event.project, event.group, event) rp = RuleProcessor( event, is_new, is_regression, is_new_group_environment, has_reappeared ) has_alert = False # TODO(dcramer): ideally this would fanout, but serializing giant # objects back and forth isn't super efficient for callback, futures in rp.apply(): has_alert = True with sentry_sdk.start_transaction( op="post_process_group", name="rule_processor_apply", sampled=True ): safe_execute(callback, event, futures) has_commit_key = "workflow-owners-ingestion:org-{}-has-commits".format( event.project.organization_id ) try: org_has_commit = cache.get(has_commit_key) if org_has_commit is None: org_has_commit = Commit.objects.filter( organization_id=event.project.organization_id ).exists() cache.set(has_commit_key, org_has_commit, 3600) if org_has_commit and features.has( "projects:workflow-owners-ingestion", event.project, ): process_suspect_commits(event=event) except Exception: logger.exception("Failed to process suspect commits") if features.has("projects:servicehooks", project=event.project): allowed_events = set(["event.created"]) if has_alert: allowed_events.add("event.alert") if allowed_events: for servicehook_id, events in _get_service_hooks(project_id=event.project_id): if any(e in allowed_events for e in events): process_service_hook.delay(servicehook_id=servicehook_id, event=event) from sentry.tasks.sentry_apps import process_resource_change_bound if event.get_event_type() == "error" and _should_send_error_created_hooks( event.project ): process_resource_change_bound.delay( action="created", sender="Error", instance_id=event.event_id, instance=event ) if is_new: process_resource_change_bound.delay( action="created", sender="Group", instance_id=event.group_id ) from sentry.plugins.base import plugins for plugin in plugins.for_project(event.project): plugin_post_process_group( plugin_slug=plugin.slug, event=event, is_new=is_new, is_regresion=is_regression ) from sentry import similarity safe_execute(similarity.record, event.project, [event]) if event.group_id: # Patch attachments that were ingested on the standalone path. update_existing_attachments(event) if not is_reprocessed: event_processed.send_robust( sender=post_process_group, project=event.project, event=event, primary_hash=kwargs.get("primary_hash"), ) with metrics.timer("tasks.post_process.delete_event_cache"): event_processing_store.delete_by_key(cache_key)
def get_plugins(project): results = [] for plugin in plugins.for_project(project, version=None): if plugin.has_project_conf(): results.append(plugin) return results
def notify(self, notification, target_type, target_identifier=None, **kwargs): metrics.incr("mail_adapter.notify") event = notification.event environment = event.get_tag("environment") group = event.group project = group.project org = group.organization subject = event.get_email_subject() query_params = {"referrer": "alert_email"} if environment: query_params["environment"] = environment link = group.get_absolute_url(params=query_params) template = "sentry/emails/error.txt" html_template = "sentry/emails/error.html" rules = [] for rule in notification.rules: rule_link = "/settings/%s/projects/%s/alerts/rules/%s/" % ( org.slug, project.slug, rule.id, ) rules.append((rule.label, rule_link)) enhanced_privacy = org.flags.enhanced_privacy # lets identify possibly suspect commits and owners commits = {} try: committers = get_serialized_event_file_committers(project, event) except (Commit.DoesNotExist, Release.DoesNotExist): pass except Exception as exc: logging.exception(six.text_type(exc)) else: for committer in committers: for commit in committer["commits"]: if commit["id"] not in commits: commit_data = commit.copy() commit_data["shortId"] = commit_data["id"][:7] commit_data["author"] = committer["author"] commit_data["subject"] = commit_data["message"].split( "\n", 1)[0] commits[commit["id"]] = commit_data project_plugins = plugins.for_project(project, version=1) organization_integrations = Integration.objects.filter( organizations=org).first() has_integrations = bool(project_plugins or organization_integrations) context = { "project_label": project.get_full_name(), "group": group, "event": event, "link": link, "rules": rules, "has_integrations": has_integrations, "enhanced_privacy": enhanced_privacy, "commits": sorted(commits.values(), key=lambda x: x["score"], reverse=True), "environment": environment, } # if the organization has enabled enhanced privacy controls we dont send # data which may show PII or source code if not enhanced_privacy: interface_list = [] for interface in six.itervalues(event.interfaces): body = interface.to_email_html(event) if not body: continue text_body = interface.to_string(event) interface_list.append( (interface.get_title(), mark_safe(body), text_body)) context.update({"tags": event.tags, "interfaces": interface_list}) headers = { "X-Sentry-Logger": group.logger, "X-Sentry-Logger-Level": group.get_level_display(), "X-Sentry-Project": project.slug, "X-Sentry-Reply-To": group_id_to_email(group.id), } for user_id in self.get_send_to( project=project, target_type=target_type, target_identifier=target_identifier, event=event, ): self.add_unsubscribe_link(context, user_id, project, "alert_email") self._send_mail( subject=subject, template=template, html_template=html_template, project=project, reference=group, headers=headers, type="notify.error", context=context, send_to=[user_id], )
def post_process_group( is_new, is_regression, is_new_group_environment, cache_key, group_id=None, **kwargs ): """ Fires post processing hooks for a group. """ from sentry.eventstore.models import Event from sentry.eventstore.processing import event_processing_store from sentry.reprocessing2 import is_reprocessed_event from sentry.utils import snuba with snuba.options_override({"consistent": True}): # We use the data being present/missing in the processing store # to ensure that we don't duplicate work should the forwarding consumers # need to rewind history. data = event_processing_store.get(cache_key) if not data: logger.info( "post_process.skipped", extra={"cache_key": cache_key, "reason": "missing_cache"}, ) return event = Event( project_id=data["project"], event_id=data["event_id"], group_id=group_id, data=data ) set_current_event_project(event.project_id) is_transaction_event = not bool(event.group_id) from sentry.models import EventDict, Organization, Project # Re-bind node data to avoid renormalization. We only want to # renormalize when loading old data from the database. event.data = EventDict(event.data, skip_renormalization=True) # Re-bind Project and Org since we're reading the Event object # from cache which may contain stale parent models. event.project = Project.objects.get_from_cache(id=event.project_id) event.project.set_cached_field_value( "organization", Organization.objects.get_from_cache(id=event.project.organization_id) ) # Simplified post processing for transaction events. # This should eventually be completely removed and transactions # will not go through any post processing. if is_transaction_event: transaction_processed.send_robust( sender=post_process_group, project=event.project, event=event, ) event_processing_store.delete_by_key(cache_key) return is_reprocessed = is_reprocessed_event(event.data) sentry_sdk.set_tag("is_reprocessed", is_reprocessed) # NOTE: we must pass through the full Event object, and not an # event_id since the Event object may not actually have been stored # in the database due to sampling. from sentry.models import Commit, GroupInboxReason from sentry.models.group import get_group_with_redirect from sentry.models.groupinbox import add_group_to_inbox from sentry.rules.processor import RuleProcessor from sentry.tasks.groupowner import process_suspect_commits from sentry.tasks.servicehooks import process_service_hook # Re-bind Group since we're reading the Event object # from cache, which may contain a stale group and project event.group, _ = get_group_with_redirect(event.group_id) event.group_id = event.group.id event.group.project = event.project event.group.project.set_cached_field_value("organization", event.project.organization) bind_organization_context(event.project.organization) _capture_stats(event, is_new) with sentry_sdk.start_span(op="tasks.post_process_group.add_group_to_inbox"): if is_reprocessed and is_new: add_group_to_inbox(event.group, GroupInboxReason.REPROCESSED) if not is_reprocessed: # we process snoozes before rules as it might create a regression # but not if it's new because you can't immediately snooze a new group has_reappeared = False if is_new else process_snoozes(event.group) if not has_reappeared: # If true, we added the .UNIGNORED reason already if is_new: add_group_to_inbox(event.group, GroupInboxReason.NEW) elif is_regression: add_group_to_inbox(event.group, GroupInboxReason.REGRESSION) with sentry_sdk.start_span(op="tasks.post_process_group.handle_owner_assignment"): handle_owner_assignment(event.project, event.group, event) rp = RuleProcessor( event, is_new, is_regression, is_new_group_environment, has_reappeared ) has_alert = False with sentry_sdk.start_span(op="tasks.post_process_group.rule_processor_callbacks"): # TODO(dcramer): ideally this would fanout, but serializing giant # objects back and forth isn't super efficient for callback, futures in rp.apply(): has_alert = True safe_execute(callback, event, futures, _with_transaction=False) try: lock = locks.get( f"w-o:{event.group_id}-d-l", duration=10, ) with lock.acquire(): has_commit_key = f"w-o:{event.project.organization_id}-h-c" org_has_commit = cache.get(has_commit_key) if org_has_commit is None: org_has_commit = Commit.objects.filter( organization_id=event.project.organization_id ).exists() cache.set(has_commit_key, org_has_commit, 3600) if org_has_commit: group_cache_key = f"w-o-i:g-{event.group_id}" if cache.get(group_cache_key): metrics.incr( "sentry.tasks.process_suspect_commits.debounce", tags={"detail": "w-o-i:g debounce"}, ) else: from sentry.utils.committers import get_frame_paths cache.set(group_cache_key, True, 604800) # 1 week in seconds event_frames = get_frame_paths(event.data) process_suspect_commits.delay( event_id=event.event_id, event_platform=event.platform, event_frames=event_frames, group_id=event.group_id, project_id=event.project_id, ) except UnableToAcquireLock: pass except Exception: logger.exception("Failed to process suspect commits") if features.has("projects:servicehooks", project=event.project): allowed_events = {"event.created"} if has_alert: allowed_events.add("event.alert") if allowed_events: for servicehook_id, events in _get_service_hooks(project_id=event.project_id): if any(e in allowed_events for e in events): process_service_hook.delay(servicehook_id=servicehook_id, event=event) from sentry.tasks.sentry_apps import process_resource_change_bound if event.get_event_type() == "error" and _should_send_error_created_hooks( event.project ): process_resource_change_bound.delay( action="created", sender="Error", instance_id=event.event_id, instance=event ) if is_new: process_resource_change_bound.delay( action="created", sender="Group", instance_id=event.group_id ) from sentry.plugins.base import plugins for plugin in plugins.for_project(event.project): plugin_post_process_group( plugin_slug=plugin.slug, event=event, is_new=is_new, is_regresion=is_regression ) from sentry import similarity with sentry_sdk.start_span(op="tasks.post_process_group.similarity"): safe_execute(similarity.record, event.project, [event], _with_transaction=False) # Patch attachments that were ingested on the standalone path. with sentry_sdk.start_span(op="tasks.post_process_group.update_existing_attachments"): update_existing_attachments(event) if not is_reprocessed: event_processed.send_robust( sender=post_process_group, project=event.project, event=event, primary_hash=kwargs.get("primary_hash"), ) with metrics.timer("tasks.post_process.delete_event_cache"): event_processing_store.delete_by_key(cache_key)
def post_process_group(event, is_new, is_regression, is_new_group_environment, **kwargs): """ Fires post processing hooks for a group. """ set_current_project(event.project_id) from sentry.utils import snuba with snuba.options_override({"consistent": True}): if check_event_already_post_processed(event): logger.info( "post_process.skipped", extra={ "project_id": event.project_id, "event_id": event.event_id, "reason": "duplicate", }, ) return # NOTE: we must pass through the full Event object, and not an # event_id since the Event object may not actually have been stored # in the database due to sampling. from sentry.models import Project, Organization, EventDict from sentry.models.group import get_group_with_redirect from sentry.rules.processor import RuleProcessor from sentry.tasks.servicehooks import process_service_hook # Re-bind node data to avoid renormalization. We only want to # renormalize when loading old data from the database. event.data = EventDict(event.data, skip_renormalization=True) if event.group_id: # Re-bind Group since we're pickling the whole Event object # which may contain a stale Project. event.group, _ = get_group_with_redirect(event.group_id) event.group_id = event.group.id # Re-bind Project and Org since we're pickling the whole Event object # which may contain stale parent models. event.project = Project.objects.get_from_cache(id=event.project_id) event.project._organization_cache = Organization.objects.get_from_cache( id=event.project.organization_id ) _capture_stats(event, is_new) if event.group_id: # we process snoozes before rules as it might create a regression # but not if it's new because you can't immediately snooze a new group has_reappeared = False if is_new else process_snoozes(event.group) handle_owner_assignment(event.project, event.group, event) rp = RuleProcessor( event, is_new, is_regression, is_new_group_environment, has_reappeared ) has_alert = False # TODO(dcramer): ideally this would fanout, but serializing giant # objects back and forth isn't super efficient for callback, futures in rp.apply(): has_alert = True with sentry_sdk.start_span( Span(op="post_process_group", transaction="rule_processor_apply", sampled=True) ): safe_execute(callback, event, futures) if features.has("projects:servicehooks", project=event.project): allowed_events = set(["event.created"]) if has_alert: allowed_events.add("event.alert") if allowed_events: for servicehook_id, events in _get_service_hooks(project_id=event.project_id): if any(e in allowed_events for e in events): process_service_hook.delay(servicehook_id=servicehook_id, event=event) from sentry.tasks.sentry_apps import process_resource_change_bound if event.get_event_type() == "error" and _should_send_error_created_hooks( event.project ): process_resource_change_bound.delay( action="created", sender="Error", instance_id=event.event_id, instance=event ) if is_new: process_resource_change_bound.delay( action="created", sender="Group", instance_id=event.group_id ) from sentry.plugins.base import plugins for plugin in plugins.for_project(event.project): plugin_post_process_group( plugin_slug=plugin.slug, event=event, is_new=is_new, is_regresion=is_regression ) event_processed.send_robust( sender=post_process_group, project=event.project, event=event, primary_hash=kwargs.get("primary_hash"), )
def post_process_group(is_new, is_regression, is_new_group_environment, cache_key, group_id=None, event=None, **kwargs): """ Fires post processing hooks for a group. """ from sentry.eventstore.models import Event from sentry.eventstore.processing import event_processing_store from sentry.utils import snuba from sentry.reprocessing2 import is_reprocessed_event with snuba.options_override({"consistent": True}): # We use the data being present/missing in the processing store # to ensure that we don't duplicate work should the forwarding consumers # need to rewind history. # # While we always send the cache_key and never send the event parameter now, # the code to handle `event` has to stick around for a self-hosted release cycle. if cache_key and event is None: data = event_processing_store.get(cache_key) if not data: logger.info( "post_process.skipped", extra={ "cache_key": cache_key, "reason": "missing_cache" }, ) return event = Event(project_id=data["project"], event_id=data["event_id"], group_id=group_id, data=data) elif event and check_event_already_post_processed(event): if cache_key: event_processing_store.delete_by_key(cache_key) logger.info( "post_process.skipped", extra={ "reason": "duplicate", "project_id": event.project_id, "event_id": event.event_id, }, ) return if is_reprocessed_event(event.data): logger.info( "post_process.skipped", extra={ "project_id": event.project_id, "event_id": event.event_id, "reason": "reprocessed", }, ) return set_current_project(event.project_id) # NOTE: we must pass through the full Event object, and not an # event_id since the Event object may not actually have been stored # in the database due to sampling. from sentry.models import Project, Organization, EventDict from sentry.models.group import get_group_with_redirect from sentry.rules.processor import RuleProcessor from sentry.tasks.servicehooks import process_service_hook # Re-bind node data to avoid renormalization. We only want to # renormalize when loading old data from the database. event.data = EventDict(event.data, skip_renormalization=True) if event.group_id: # Re-bind Group since we're reading the Event object # from cache, which may contain a stale group and project event.group, _ = get_group_with_redirect(event.group_id) event.group_id = event.group.id # Re-bind Project and Org since we're reading the Event object # from cache which may contain stale parent models. event.project = Project.objects.get_from_cache(id=event.project_id) event.project._organization_cache = Organization.objects.get_from_cache( id=event.project.organization_id) bind_organization_context(event.project.organization) _capture_stats(event, is_new) if event.group_id: # we process snoozes before rules as it might create a regression # but not if it's new because you can't immediately snooze a new group has_reappeared = False if is_new else process_snoozes(event.group) handle_owner_assignment(event.project, event.group, event) rp = RuleProcessor(event, is_new, is_regression, is_new_group_environment, has_reappeared) has_alert = False # TODO(dcramer): ideally this would fanout, but serializing giant # objects back and forth isn't super efficient for callback, futures in rp.apply(): has_alert = True with sentry_sdk.start_transaction(op="post_process_group", name="rule_processor_apply", sampled=True): safe_execute(callback, event, futures) if features.has("projects:servicehooks", project=event.project): allowed_events = set(["event.created"]) if has_alert: allowed_events.add("event.alert") if allowed_events: for servicehook_id, events in _get_service_hooks( project_id=event.project_id): if any(e in allowed_events for e in events): process_service_hook.delay( servicehook_id=servicehook_id, event=event) from sentry.tasks.sentry_apps import process_resource_change_bound if event.get_event_type( ) == "error" and _should_send_error_created_hooks(event.project): process_resource_change_bound.delay(action="created", sender="Error", instance_id=event.event_id, instance=event) if is_new: process_resource_change_bound.delay(action="created", sender="Group", instance_id=event.group_id) # Patch attachments that were ingested on the standalone path. update_existing_attachments(event) from sentry.plugins.base import plugins for plugin in plugins.for_project(event.project): plugin_post_process_group(plugin_slug=plugin.slug, event=event, is_new=is_new, is_regresion=is_regression) event_processed.send_robust( sender=post_process_group, project=event.project, event=event, primary_hash=kwargs.get("primary_hash"), ) with metrics.timer("tasks.post_process.delete_event_cache"): event_processing_store.delete_by_key(cache_key)