def process_profile(profile: MutableMapping[str, Any], **kwargs: Any) -> None: if profile["platform"] == "cocoa": if not _validate_ios_profile(profile=profile): return None profile = _symbolicate(profile=profile) profile = _normalize(profile=profile) global processed_profiles_publisher if processed_profiles_publisher is None: config = settings.KAFKA_TOPICS[settings.KAFKA_PROFILES] processed_profiles_publisher = KafkaPublisher( kafka_config.get_kafka_producer_cluster_options( config["cluster"]), ) processed_profiles_publisher.publish( "processed-profiles", json.dumps(profile), )
def track_outcome( org_id, project_id, key_id, outcome, reason=None, timestamp=None, event_id=None, category=None, quantity=None, ): """ This is a central point to track org/project counters per incoming event. NB: This should only ever be called once per incoming event, which means it should only be called at the point we know the final outcome for the event (invalid, rate_limited, accepted, discarded, etc.) This increments all the relevant legacy RedisTSDB counters, as well as sending a single metric event to Kafka which can be used to reconstruct the counters with SnubaTSDB. """ global outcomes_publisher if outcomes_publisher is None: outcomes_publisher = KafkaPublisher(settings.KAFKA_CLUSTERS[outcomes["cluster"]]) if quantity is None: quantity = 1 assert isinstance(org_id, six.integer_types) assert isinstance(project_id, six.integer_types) assert isinstance(key_id, (type(None), six.integer_types)) assert isinstance(outcome, Outcome) assert isinstance(timestamp, (type(None), datetime)) assert isinstance(category, (type(None), DataCategory)) assert isinstance(quantity, int) timestamp = timestamp or to_datetime(time.time()) tsdb_in_consumer = decide_tsdb_in_consumer() if not tsdb_in_consumer: increment_list = list( tsdb_increments_from_outcome( org_id=org_id, project_id=project_id, key_id=key_id, outcome=outcome, reason=reason, category=category, ) ) if increment_list: tsdb.incr_multi(increment_list, timestamp=timestamp) if project_id and event_id: mark_tsdb_incremented(project_id, event_id) # Send a snuba metrics payload. outcomes_publisher.publish( outcomes["topic"], json.dumps( { "timestamp": timestamp, "org_id": org_id, "project_id": project_id, "key_id": key_id, "outcome": outcome.value, "reason": reason, "event_id": event_id, "category": category, "quantity": quantity, } ), ) metrics.incr( "events.outcomes", skip_internal=True, tags={"outcome": outcome.name.lower(), "reason": reason}, )
def track_outcome( org_id, project_id, key_id, outcome, reason=None, timestamp=None, event_id=None, category=None, quantity=None, ): """ This is a central point to track org/project counters per incoming event. NB: This should only ever be called once per incoming event, which means it should only be called at the point we know the final outcome for the event (invalid, rate_limited, accepted, discarded, etc.) This sends the "outcome" message to Kafka which is used by Snuba to serve data for SnubaTSDB and RedisSnubaTSDB, such as # of rate-limited/filtered events. """ global outcomes_publisher if outcomes_publisher is None: cluster_name = outcomes["cluster"] outcomes_publisher = KafkaPublisher( kafka_config.get_kafka_producer_cluster_options(cluster_name)) if quantity is None: quantity = 1 assert isinstance(org_id, int) assert isinstance(project_id, int) assert isinstance(key_id, (type(None), int)) assert isinstance(outcome, Outcome) assert isinstance(timestamp, (type(None), datetime)) assert isinstance(category, (type(None), DataCategory)) assert isinstance(quantity, int) timestamp = timestamp or to_datetime(time.time()) # Send a snuba metrics payload. outcomes_publisher.publish( outcomes["topic"], json.dumps({ "timestamp": timestamp, "org_id": org_id, "project_id": project_id, "key_id": key_id, "outcome": outcome.value, "reason": reason, "event_id": event_id, "category": category, "quantity": quantity, }), ) metrics.incr( "events.outcomes", skip_internal=True, tags={ "outcome": outcome.name.lower(), "reason": reason, "category": category.api_name() if category is not None else "null", }, )