def process_profile(profile: MutableMapping[str, Any], **kwargs: Any) -> None: if profile["platform"] == "cocoa": if not _validate_ios_profile(profile=profile): return None profile = _symbolicate(profile=profile) profile = _normalize(profile=profile) global processed_profiles_publisher if processed_profiles_publisher is None: config = settings.KAFKA_TOPICS[settings.KAFKA_PROFILES] processed_profiles_publisher = KafkaPublisher( kafka_config.get_kafka_producer_cluster_options( config["cluster"]), ) processed_profiles_publisher.publish( "processed-profiles", json.dumps(profile), )
from sentry.web.client_config import get_client_config from sentry.relay.config import get_project_config logger = logging.getLogger("sentry") minidumps_logger = logging.getLogger("sentry.minidumps") # Transparent 1x1 gif # See http://probablyprogramming.com/2009/03/15/the-tiniest-gif-ever PIXEL = base64.b64decode("R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs=") PROTOCOL_VERSIONS = frozenset(("2.0", "3", "4", "5", "6", "7")) kafka_publisher = ( QueuedPublisherService( KafkaPublisher( getattr(settings, "KAFKA_RAW_EVENTS_PUBLISHER_CONNECTION", None), asynchronous=False ) ) if getattr(settings, "KAFKA_RAW_EVENTS_PUBLISHER_ENABLED", False) else None ) def allow_cors_options(func): """ Decorator that adds automatic handling of OPTIONS requests for CORS If the request is OPTIONS (i.e. pre flight CORS) construct a OK (200) response in which we explicitly enable the caller and add the custom headers that we support For other requests just add the appropriate CORS headers
def track_outcome( org_id, project_id, key_id, outcome, reason=None, timestamp=None, event_id=None, category=None, quantity=None, ): """ This is a central point to track org/project counters per incoming event. NB: This should only ever be called once per incoming event, which means it should only be called at the point we know the final outcome for the event (invalid, rate_limited, accepted, discarded, etc.) This increments all the relevant legacy RedisTSDB counters, as well as sending a single metric event to Kafka which can be used to reconstruct the counters with SnubaTSDB. """ global outcomes_publisher if outcomes_publisher is None: outcomes_publisher = KafkaPublisher(settings.KAFKA_CLUSTERS[outcomes["cluster"]]) if quantity is None: quantity = 1 assert isinstance(org_id, six.integer_types) assert isinstance(project_id, six.integer_types) assert isinstance(key_id, (type(None), six.integer_types)) assert isinstance(outcome, Outcome) assert isinstance(timestamp, (type(None), datetime)) assert isinstance(category, (type(None), DataCategory)) assert isinstance(quantity, int) timestamp = timestamp or to_datetime(time.time()) tsdb_in_consumer = decide_tsdb_in_consumer() if not tsdb_in_consumer: increment_list = list( tsdb_increments_from_outcome( org_id=org_id, project_id=project_id, key_id=key_id, outcome=outcome, reason=reason, category=category, ) ) if increment_list: tsdb.incr_multi(increment_list, timestamp=timestamp) if project_id and event_id: mark_tsdb_incremented(project_id, event_id) # Send a snuba metrics payload. outcomes_publisher.publish( outcomes["topic"], json.dumps( { "timestamp": timestamp, "org_id": org_id, "project_id": project_id, "key_id": key_id, "outcome": outcome.value, "reason": reason, "event_id": event_id, "category": category, "quantity": quantity, } ), ) metrics.incr( "events.outcomes", skip_internal=True, tags={"outcome": outcome.name.lower(), "reason": reason}, )
def track_outcome(org_id, project_id, key_id, outcome, reason=None, timestamp=None, event_id=None): """ This is a central point to track org/project counters per incoming event. NB: This should only ever be called once per incoming event, which means it should only be called at the point we know the final outcome for the event (invalid, rate_limited, accepted, discarded, etc.) This increments all the relevant legacy RedisTSDB counters, as well as sending a single metric event to Kafka which can be used to reconstruct the counters with SnubaTSDB. """ global outcomes_publisher if outcomes_publisher is None: outcomes_publisher = QueuedPublisherService( KafkaPublisher(settings.KAFKA_CLUSTERS[outcomes["cluster"]])) assert isinstance(org_id, six.integer_types) assert isinstance(project_id, six.integer_types) assert isinstance(key_id, (type(None), six.integer_types)) assert isinstance(outcome, Outcome) assert isinstance(timestamp, (type(None), datetime)) timestamp = timestamp or to_datetime(time.time()) increment_list = [] if outcome != Outcome.INVALID: # This simply preserves old behavior. We never counted invalid events # (too large, duplicate, CORS) toward regular `received` counts. increment_list.extend([ (tsdb.models.project_total_received, project_id), (tsdb.models.organization_total_received, org_id), (tsdb.models.key_total_received, key_id), ]) if outcome == Outcome.FILTERED: increment_list.extend([ (tsdb.models.project_total_blacklisted, project_id), (tsdb.models.organization_total_blacklisted, org_id), (tsdb.models.key_total_blacklisted, key_id), ]) elif outcome == Outcome.RATE_LIMITED: increment_list.extend([ (tsdb.models.project_total_rejected, project_id), (tsdb.models.organization_total_rejected, org_id), (tsdb.models.key_total_rejected, key_id), ]) if reason in FILTER_STAT_KEYS_TO_VALUES: increment_list.append((FILTER_STAT_KEYS_TO_VALUES[reason], project_id)) increment_list = [(model, key) for model, key in increment_list if key is not None] if increment_list: tsdb.incr_multi(increment_list, timestamp=timestamp) # Send a snuba metrics payload. outcomes_publisher.publish( outcomes["topic"], json.dumps({ "timestamp": timestamp, "org_id": org_id, "project_id": project_id, "key_id": key_id, "outcome": outcome.value, "reason": reason, "event_id": event_id, }), ) metrics.incr( "events.outcomes", skip_internal=True, tags={ "outcome": outcome.name.lower(), "reason": reason }, )
def track_outcome( org_id, project_id, key_id, outcome, reason=None, timestamp=None, event_id=None, category=None, quantity=None, ): """ This is a central point to track org/project counters per incoming event. NB: This should only ever be called once per incoming event, which means it should only be called at the point we know the final outcome for the event (invalid, rate_limited, accepted, discarded, etc.) This sends the "outcome" message to Kafka which is used by Snuba to serve data for SnubaTSDB and RedisSnubaTSDB, such as # of rate-limited/filtered events. """ global outcomes_publisher if outcomes_publisher is None: cluster_name = outcomes["cluster"] outcomes_publisher = KafkaPublisher( kafka_config.get_kafka_producer_cluster_options(cluster_name)) if quantity is None: quantity = 1 assert isinstance(org_id, int) assert isinstance(project_id, int) assert isinstance(key_id, (type(None), int)) assert isinstance(outcome, Outcome) assert isinstance(timestamp, (type(None), datetime)) assert isinstance(category, (type(None), DataCategory)) assert isinstance(quantity, int) timestamp = timestamp or to_datetime(time.time()) # Send a snuba metrics payload. outcomes_publisher.publish( outcomes["topic"], json.dumps({ "timestamp": timestamp, "org_id": org_id, "project_id": project_id, "key_id": key_id, "outcome": outcome.value, "reason": reason, "event_id": event_id, "category": category, "quantity": quantity, }), ) metrics.incr( "events.outcomes", skip_internal=True, tags={ "outcome": outcome.name.lower(), "reason": reason, "category": category.api_name() if category is not None else "null", }, )
def track_outcome(org_id, project_id, key_id, outcome, reason=None, timestamp=None): """ This is a central point to track org/project counters per incoming event. NB: This should only ever be called once per incoming event, which means it should only be called at the point we know the final outcome for the event (invalid, rate_limited, accepted, discarded, etc.) This increments all the relevant legacy RedisTSDB counters, as well as sending a single metric event to Kafka which can be used to reconstruct the counters with SnubaTSDB. """ global outcomes_publisher if outcomes_publisher is None: outcomes_publisher = QueuedPublisherService( KafkaPublisher(settings.KAFKA_CLUSTERS[outcomes['cluster']])) timestamp = timestamp or to_datetime(time.time()) increment_list = [] if outcome != 'invalid': # This simply preserves old behavior. We never counted invalid events # (too large, duplicate, CORS) toward regular `received` counts. increment_list.extend([ (tsdb.models.project_total_received, project_id), (tsdb.models.organization_total_received, org_id), (tsdb.models.key_total_received, key_id), ]) if outcome == 'filtered': increment_list.extend([ (tsdb.models.project_total_blacklisted, project_id), (tsdb.models.organization_total_blacklisted, org_id), (tsdb.models.key_total_blacklisted, key_id), ]) elif outcome == 'rate_limited': increment_list.extend([ (tsdb.models.project_total_rejected, project_id), (tsdb.models.organization_total_rejected, org_id), (tsdb.models.key_total_rejected, key_id), ]) if reason in FILTER_STAT_KEYS_TO_VALUES: increment_list.append((FILTER_STAT_KEYS_TO_VALUES[reason], project_id)) increment_list = [(model, key) for model, key in increment_list if key is not None] if increment_list: tsdb.incr_multi(increment_list, timestamp=timestamp) # Send a snuba metrics payload. if random.random() <= options.get('snuba.track-outcomes-sample-rate'): outcomes_publisher.publish( outcomes['topic'], json.dumps({ 'timestamp': timestamp, 'org_id': org_id, 'project_id': project_id, 'key_id': key_id, 'outcome': outcome, 'reason': reason, })) metrics.incr( 'events.outcomes', skip_internal=True, tags={ 'outcome': outcome, 'reason': reason, }, )
def track_outcome( org_id: int, project_id: int, key_id: Optional[int], outcome: Outcome, reason: Optional[str] = None, timestamp: Optional[datetime] = None, event_id: Optional[str] = None, category: Optional[DataCategory] = None, quantity: Optional[int] = None, ) -> None: """ This is a central point to track org/project counters per incoming event. NB: This should only ever be called once per incoming event, which means it should only be called at the point we know the final outcome for the event (invalid, rate_limited, accepted, discarded, etc.) This sends the "outcome" message to Kafka which is used by Snuba to serve data for SnubaTSDB and RedisSnubaTSDB, such as # of rate-limited/filtered events. """ global outcomes_publisher global billing_publisher if quantity is None: quantity = 1 assert isinstance(org_id, int) assert isinstance(project_id, int) assert isinstance(key_id, (type(None), int)) assert isinstance(outcome, Outcome) assert isinstance(timestamp, (type(None), datetime)) assert isinstance(category, (type(None), DataCategory)) assert isinstance(quantity, int) outcomes_config = settings.KAFKA_TOPICS[settings.KAFKA_OUTCOMES] billing_config = settings.KAFKA_TOPICS.get(settings.KAFKA_OUTCOMES_BILLING) or outcomes_config # Create a second producer instance only if the cluster differs. Otherwise, # reuse the same producer and just send to the other topic. if outcome.is_billing() and billing_config["cluster"] != outcomes_config["cluster"]: if billing_publisher is None: cluster_name = billing_config["cluster"] billing_publisher = KafkaPublisher( kafka_config.get_kafka_producer_cluster_options(cluster_name) ) publisher = billing_publisher else: if outcomes_publisher is None: cluster_name = outcomes_config["cluster"] outcomes_publisher = KafkaPublisher( kafka_config.get_kafka_producer_cluster_options(cluster_name) ) publisher = outcomes_publisher timestamp = timestamp or to_datetime(time.time()) # Send billing outcomes to a dedicated topic if there is a separate # configuration for it. Otherwise, fall back to the regular outcomes topic. # This does NOT switch the producer, if both topics are on the same cluster. # # In Sentry, there is no significant difference between the classes of # outcome. In Sentry SaaS, they have elevated stability requirements as they # are used for spike protection and quota enforcement. topic_name = billing_config["topic"] if outcome.is_billing() else outcomes_config["topic"] # Send a snuba metrics payload. publisher.publish( topic_name, json.dumps( { "timestamp": timestamp, "org_id": org_id, "project_id": project_id, "key_id": key_id, "outcome": outcome.value, "reason": reason, "event_id": event_id, "category": category, "quantity": quantity, } ), ) metrics.incr( "events.outcomes", skip_internal=True, tags={ "outcome": outcome.name.lower(), "reason": reason, "category": category.api_name() if category is not None else "null", "topic": topic_name, }, )