flatten_nested_field, ) from snuba.processor import ( InsertBatch, MessageProcessor, ProcessedMessage, _as_dict_safe, _ensure_valid_date, _ensure_valid_ip, _unicodify, ) from snuba.utils.metrics.wrapper import MetricsWrapper logger = logging.getLogger(__name__) metrics = MetricsWrapper(environment.metrics, "transactions.processor") UNKNOWN_SPAN_STATUS = 2 class TransactionsMessageProcessor(MessageProcessor): PROMOTED_TAGS = { "environment", "sentry:release", "sentry:user", "sentry:dist", } def __extract_timestamp(self, field): timestamp = _ensure_valid_date(datetime.fromtimestamp(field)) if timestamp is None:
from snuba import environment from snuba.datasets.events_format import enforce_retention, extract_extra_tags from snuba.processor import ( InsertBatch, MessageProcessor, ProcessedMessage, _as_dict_safe, _ensure_valid_date, _unicodify, ) from snuba.utils.metrics.wrapper import MetricsWrapper UNKNOWN_SPAN_STATUS = 2 metrics = MetricsWrapper(environment.metrics, "spans.processor") class SpansMessageProcessor(MessageProcessor): def __extract_timestamp(self, field: float) -> Tuple[datetime, int]: timestamp = _ensure_valid_date(datetime.fromtimestamp(field)) if timestamp is None: timestamp = datetime.utcnow() nanoseconds = int(timestamp.microsecond * 1000) return (timestamp, nanoseconds) def __init_span(self, event: Mapping[str, Any]) -> MutableMapping[str, Any]: """ Initializes the fields that are the same for all spans within a transaction. """
from snuba.request.request_settings import HTTPRequestSettings, RequestSettings from snuba.request.schema import RequestParts, RequestSchema from snuba.request.validation import build_request, parse_legacy_query, parse_snql_query from snuba.state.rate_limit import RateLimitExceeded from snuba.subscriptions.codecs import SubscriptionDataCodec from snuba.subscriptions.data import InvalidSubscriptionError, PartitionId from snuba.subscriptions.subscription import SubscriptionCreator, SubscriptionDeleter from snuba.util import with_span from snuba.utils.metrics.timer import Timer from snuba.utils.metrics.wrapper import MetricsWrapper from snuba.web import QueryException from snuba.web.converters import DatasetConverter from snuba.web.query import parse_and_run_query from snuba.writer import BatchWriterEncoderWrapper, WriterTableRow metrics = MetricsWrapper(environment.metrics, "api") logger = logging.getLogger("snuba.api") # Flask wants a Dict, not a Mapping RespTuple = Tuple[Text, int, Dict[Any, Any]] try: import uwsgi except ImportError: def check_down_file_exists() -> bool: return False else:
from snuba.query.matchers import FunctionCall as FunctionCallMatch from snuba.query.matchers import String as StringMatch from snuba.query.parser.conditions import parse_conditions_to_expr from snuba.query.parser.exceptions import ( AliasShadowingException, CyclicAliasException, ParsingException, ) from snuba.query.parser.expressions import parse_aggregation, parse_expression from snuba.query.parser.validation import validate_query from snuba.util import is_function, to_list, tuplify from snuba.utils.metrics.wrapper import MetricsWrapper logger = logging.getLogger(__name__) metrics = MetricsWrapper(environment.metrics, "parser") def parse_query(body: MutableMapping[str, Any], dataset: Dataset) -> Query: """ Parses the query body generating the AST. This only takes into account the initial query body. Extensions are parsed by extension processors and are supposed to update the AST. Parsing includes two phases. The first transforms the json body into a minimal query Object resolving expressions, conditions, etc. The second phase performs some query processing to provide a sane query to the dataset specific section. - It prevents alias shadowing. - It transforms columns from the tags[asd] form into SubscriptableReference.
def __init__( self, storage_key: StorageKey, raw_topic: Optional[str], replacements_topic: Optional[str], max_batch_size: int, max_batch_time_ms: int, bootstrap_servers: Sequence[str], group_id: str, commit_log_topic: Optional[str], auto_offset_reset: str, queued_max_messages_kbytes: int, queued_min_messages: int, strategy_factory_type: StrategyFactoryType, processes: Optional[int], input_block_size: Optional[int], output_block_size: Optional[int], commit_retry_policy: Optional[RetryPolicy] = None, profile_path: Optional[str] = None, ) -> None: self.storage = get_writable_storage(storage_key) self.bootstrap_servers = bootstrap_servers stream_loader = self.storage.get_table_writer().get_stream_loader() self.raw_topic: Topic if raw_topic is not None: self.raw_topic = Topic(raw_topic) else: self.raw_topic = Topic(stream_loader.get_default_topic_spec().topic_name) self.replacements_topic: Optional[Topic] if replacements_topic is not None: self.replacements_topic = Topic(replacements_topic) else: replacement_topic_spec = stream_loader.get_replacement_topic_spec() if replacement_topic_spec is not None: self.replacements_topic = Topic(replacement_topic_spec.topic_name) else: self.replacements_topic = None self.commit_log_topic: Optional[Topic] if commit_log_topic is not None: self.commit_log_topic = Topic(commit_log_topic) else: commit_log_topic_spec = stream_loader.get_commit_log_topic_spec() if commit_log_topic_spec is not None: self.commit_log_topic = Topic(commit_log_topic_spec.topic_name) else: self.commit_log_topic = None # XXX: This can result in a producer being built in cases where it's # not actually required. self.producer = Producer( { "bootstrap.servers": ",".join(self.bootstrap_servers), "partitioner": "consistent", "message.max.bytes": 50000000, # 50MB, default is 1MB } ) self.metrics = MetricsWrapper( environment.metrics, "consumer", tags={"group": group_id, "storage": storage_key.value}, ) self.max_batch_size = max_batch_size self.max_batch_time_ms = max_batch_time_ms self.group_id = group_id self.auto_offset_reset = auto_offset_reset self.queued_max_messages_kbytes = queued_max_messages_kbytes self.queued_min_messages = queued_min_messages self.strategy_factory_type = strategy_factory_type self.processes = processes self.input_block_size = input_block_size self.output_block_size = output_block_size self.__profile_path = profile_path if ( self.processes is not None and self.strategy_factory_type is not StrategyFactoryType.STREAMING ): raise ValueError( "process count can only be specified when using streaming strategy" ) if commit_retry_policy is None: commit_retry_policy = BasicRetryPolicy( 3, constant_delay(1), lambda e: isinstance(e, KafkaException) and e.args[0].code() in ( KafkaError.REQUEST_TIMED_OUT, KafkaError.NOT_COORDINATOR_FOR_GROUP, KafkaError._WAIT_COORD, ), ) self.__commit_retry_policy = commit_retry_policy
RateLimitAggregator, RateLimitExceeded, RateLimitStatsContainer, get_global_rate_limit_params, ) from snuba.util import force_bytes, with_span from snuba.utils.codecs import ExceptionAwareCodec from snuba.utils.metrics.timer import Timer from snuba.utils.metrics.wrapper import MetricsWrapper from snuba.utils.serializable_exception import ( SerializableException, SerializableExceptionDict, ) from snuba.web import QueryException, QueryResult metrics = MetricsWrapper(environment.metrics, "db_query") class ResultCacheCodec(ExceptionAwareCodec[bytes, Result]): def encode(self, value: Result) -> bytes: return cast(str, rapidjson.dumps(value)).encode("utf-8") def decode(self, value: bytes) -> Result: ret = rapidjson.loads(value) if ret.get("__type__", "DNE") == "SerializableException": raise SerializableException.from_dict(cast(SerializableExceptionDict, ret)) if not isinstance(ret, Mapping) or "meta" not in ret or "data" not in ret: raise ValueError("Invalid value type in result cache") return cast(Result, ret) def encode_exception(self, value: SerializableException) -> bytes:
def multistorage_consumer( storage_names: Sequence[str], consumer_group: str, max_batch_size: int, max_batch_time_ms: int, auto_offset_reset: str, queued_max_messages_kbytes: int, queued_min_messages: int, processes: int, input_block_size: int, output_block_size: int, log_level: Optional[str] = None, ) -> None: setup_logging(log_level) setup_sentry() storages = { key: get_writable_storage(key) for key in (getattr(StorageKey, name.upper()) for name in storage_names) } topics = { storage.get_table_writer() .get_stream_loader() .get_default_topic_spec() .topic_name for storage in storages.values() } # XXX: The ``StreamProcessor`` only supports a single topic at this time, # but is easily modified. The topic routing in the processing strategy is a # bit trickier (but also shouldn't be too bad.) topic = Topic(topics.pop()) if topics: raise ValueError("only one topic is supported") # XXX: The ``CommitLogConsumer`` also only supports a single topic at this # time. (It is less easily modified.) This also assumes the commit log # topic is on the same Kafka cluster as the input topic. commit_log_topics = { spec.topic_name for spec in ( storage.get_table_writer().get_stream_loader().get_commit_log_topic_spec() for storage in storages.values() ) if spec is not None } commit_log_topic: Optional[Topic] if commit_log_topics: commit_log_topic = Topic(commit_log_topics.pop()) else: commit_log_topic = None if commit_log_topics: raise ValueError("only one commit log topic is supported") # XXX: This requires that all storages are associated with the same Kafka # cluster so that they can be consumed by the same consumer instance. # Unfortunately, we don't have the concept of independently configurable # Kafka clusters in settings, only consumer configurations that are # associated with storages and/or global default configurations. To avoid # implementing yet another method of configuring Kafka clusters, this just # piggybacks on the existing configuration method(s), with the assumption # that most deployments are going to be using the default configuration. storage_keys = [*storages.keys()] consumer_configuration = build_kafka_consumer_configuration( storage_keys[0], consumer_group, auto_offset_reset=auto_offset_reset, queued_max_messages_kbytes=queued_max_messages_kbytes, queued_min_messages=queued_min_messages, ) for storage_key in storage_keys[1:]: if ( build_kafka_consumer_configuration(storage_key, consumer_group)[ "bootstrap.servers" ] != consumer_configuration["bootstrap.servers"] ): raise ValueError("storages cannot be located on different Kafka clusters") if commit_log_topic is None: consumer = KafkaConsumer(consumer_configuration) else: # XXX: This relies on the assumptions that a.) the Kafka cluster where # the commit log topic is located is the same as the input topic (there # is no way to specify otherwise, at writing) and b.) all storages are # located on the same Kafka cluster (validated above.) producer = ConfluentKafkaProducer( build_kafka_producer_configuration(storage_keys[0]) ) consumer = KafkaConsumerWithCommitLog( consumer_configuration, producer=producer, commit_log_topic=commit_log_topic, ) metrics = MetricsWrapper(environment.metrics, "consumer") processor = StreamProcessor( consumer, topic, MultistorageConsumerProcessingStrategyFactory( [*storages.values()], max_batch_size, max_batch_time_ms / 1000.0, processes=processes, input_block_size=input_block_size, output_block_size=output_block_size, metrics=metrics, ), metrics=metrics, ) def handler(signum: int, frame: Any) -> None: processor.signal_shutdown() signal.signal(signal.SIGINT, handler) signal.signal(signal.SIGTERM, handler) processor.run()
cast, ) from uuid import UUID from sentry_sdk import set_tag from snuba import environment, settings from snuba.query.logical import Query from snuba.query.query_settings import QuerySettings from snuba.reader import Row from snuba.state import get_config from snuba.utils.metrics.wrapper import MetricsWrapper from snuba.utils.threaded_function_delegator import Result from snuba.web import QueryResult metrics = MetricsWrapper(environment.metrics, "query.similarity") logger = logging.getLogger("snuba.upgrade_discrepancies") class Option(Enum): ERRORS = 1 ERRORS_V2 = 2 TRANSACTIONS = 3 TRANSACTIONS_V2 = 4 class Choice(NamedTuple): primary: Option secondary: Optional[Option]
from snuba.clickhouse.query import Query from snuba.query.matchers import ( AnyOptionalString, Or, Param, String, ) from snuba.query.matchers import Column as ColumnMatch from snuba.query.matchers import FunctionCall as FunctionCallMatch from snuba.query.matchers import Literal as LiteralMatch from snuba.clickhouse.processors import QueryProcessor from snuba.request.request_settings import RequestSettings from snuba.utils.metrics.wrapper import MetricsWrapper metrics = MetricsWrapper(environment.metrics, "api.query.uuid_processor") class UUIDColumnProcessor(QueryProcessor): """ If a condition is being performed on a column that stores UUIDs (as defined in the constructor) then change the condition to use a proper UUID instead of a string. """ def formatted_uuid_pattern(self, suffix: str = "") -> FunctionCallMatch: return FunctionCallMatch( String("replaceAll"), ( FunctionCallMatch( String("toString"), (
def __init__(self) -> None: super().__init__(default_entity=EntityKey.DISCOVER) # XXX: This is temporary code that will eventually need to be ported to Sentry # since SnQL will require an entity to always be specified by the user. def select_entity(self, query: Query) -> EntityKey: selected_entity = match_query_to_entity( query, EVENTS_COLUMNS, TRANSACTIONS_COLUMNS ) track_bad_query(query, selected_entity, EVENTS_COLUMNS, TRANSACTIONS_COLUMNS) return selected_entity metrics = MetricsWrapper(environment.metrics, "api.discover") logger = logging.getLogger(__name__) EVENT_CONDITION = FunctionCallMatch( Param("function", Or([StringMatch(op) for op in BINARY_OPERATORS])), ( Or([ColumnMatch(None, StringMatch("type")), LiteralMatch(None)]), Param("event_type", Or([ColumnMatch(), LiteralMatch()])), ), ) TRANSACTION_FUNCTIONS = FunctionCallMatch( Or([StringMatch("apdex"), StringMatch("failure_rate")]), None )
extract_user, ) from snuba.processor import ( InsertBatch, MessageProcessor, ProcessedMessage, _as_dict_safe, _ensure_valid_date, _ensure_valid_ip, _unicodify, ) from snuba.utils.metrics.wrapper import MetricsWrapper logger = logging.getLogger(__name__) metrics = MetricsWrapper(environment.metrics, "replays.processor") from sentry_sdk import capture_exception EventDict = Mapping[Any, Any] RetentionDays = int class ReplaysProcessor(MessageProcessor): PROMOTED_TAGS = { "environment", "sentry:release", "sentry:user", "sentry:dist", } def __extract_timestamp(self, field: int) -> datetime: