def test_tags_hash_map(self) -> None: """ Adds an event and ensures the tags_hash_map is properly populated including escaping. """ self.event = get_raw_event() self.event["data"]["tags"].append(["test_tag1", "value1"]) self.event["data"]["tags"].append(["test_tag=2", "value2"]) # Requires escaping self.write_unprocessed_events([self.event]) clickhouse = ( get_storage(StorageKey.EVENTS) .get_cluster() .get_query_connection(ClickhouseClientSettings.QUERY) ) hashed = clickhouse.execute( "SELECT cityHash64('test_tag1=value1'), cityHash64('test_tag\\\\=2=value2')" ) tag1, tag2 = hashed[0] event = clickhouse.execute( ( f"SELECT event_id FROM sentry_local WHERE has(_tags_hash_map, {tag1}) " f"AND has(_tags_hash_map, {tag2})" ) ) assert len(event) == 1 assert event[0][0] == self.event["data"]["id"]
def setup_method(self, test_method): self.metadata = KafkaMessageMetadata(0, 0, datetime.now()) self.event = get_raw_event() self.processor = (get_writable_storage( StorageKey.EVENTS).get_table_writer().get_stream_loader(). get_processor())
def test_tags_hash_map(self) -> None: """ Adds an event and ensures the tags_hash_map is properly populated including escaping. """ self.event = get_raw_event() self.event["data"]["tags"].append(["test_tag1", "value1"]) self.event["data"]["tags"].append(["test_tag=2", "value2"]) # Requires escaping storage = get_writable_storage(StorageKey.ERRORS) schema = storage.get_schema() assert isinstance(schema, TableSchema) table_name = schema.get_table_name() write_unprocessed_events(storage, [self.event]) clickhouse = storage.get_cluster().get_query_connection( ClickhouseClientSettings.QUERY) hashed = clickhouse.execute( "SELECT cityHash64('test_tag1=value1'), cityHash64('test_tag\\\\=2=value2')" ) tag1, tag2 = hashed[0] event = clickhouse.execute(( f"SELECT replaceAll(toString(event_id), '-', '') FROM {table_name} WHERE has(_tags_hash_map, {tag1}) " f"AND has(_tags_hash_map, {tag2})")) assert len(event) == 1 assert event[0][0] == self.event["data"]["id"]
def setup_method(self, test_method): super().setup_method(test_method) self.app.post = partial(self.app.post, headers={"referer": "test"}) self.trace_id = uuid.UUID("7400045b-25c4-43b8-8591-4600aa83ad04") self.event = get_raw_event() self.project_id = self.event["project_id"] write_unprocessed_events(get_writable_storage(StorageKey.EVENTS), [self.event]) write_unprocessed_events( get_writable_storage(StorageKey.TRANSACTIONS), [get_raw_transaction()], )
def setup_method(self): from snuba.web.views import application assert application.testing is True self.app = application.test_client() self.app.post = partial(self.app.post, headers={"referer": "test"}) self.storage = get_storage(StorageKey.EVENTS) self.replacer = replacer.ReplacerWorker( self.storage, DummyMetricsBackend(strict=True)) self.project_id = 1 self.event = get_raw_event()
def setup_method(self) -> None: from snuba.web.views import application assert application.testing is True self.app = application.test_client() self.storage = get_writable_storage(StorageKey.ERRORS) self.replacer = replacer.ReplacerWorker( self.storage, CONSUMER_GROUP, DummyMetricsBackend(strict=True), ) self.project_id = 1 self.event = get_raw_event()
def setup_method(self, test_method): super().setup_method(test_method) self.app.post = partial(self.app.post, headers={"referer": "test"}) self.trace_id = uuid.UUID("7400045b-25c4-43b8-8591-4600aa83ad04") self.event = get_raw_event() self.project_id = self.event["project_id"] self.skew = timedelta(minutes=180) self.base_time = datetime.utcnow().replace( minute=0, second=0, microsecond=0, tzinfo=pytz.utc) - timedelta(minutes=180) write_unprocessed_events(get_writable_storage(StorageKey.EVENTS), [self.event]) write_unprocessed_events( get_writable_storage(StorageKey.TRANSACTIONS), [get_raw_transaction()], )
def setup_method(self, test_method): super(TestReplacer, self).setup_method(test_method, "events_migration") from snuba.web.views import application assert application.testing is True self.app = application.test_client() self.app.post = partial(self.app.post, headers={"referer": "test"}) self.replacer = replacer.ReplacerWorker( self.dataset.get_writable_storage(), DummyMetricsBackend(strict=True)) self.project_id = 1 self.event = get_raw_event()
def setup_method(self): from snuba.web.views import application assert application.testing is True self.app = application.test_client() self.app.post = partial(self.app.post, headers={"referer": "test"}) self.storage = get_writable_storage(StorageKey.ERRORS) self.replacer = replacer.ReplacerWorker( self.storage, DummyMetricsBackend(strict=True)) self.project_id = 1 self.event = get_raw_event() settings.ERRORS_ROLLOUT_ALL = True settings.ERRORS_ROLLOUT_WRITABLE_STORAGE = True
def test_backfill_errors() -> None: backfill_migration_id = "0014_backfill_errors" runner = Runner() runner.run_migration(MigrationKey(MigrationGroup.SYSTEM, "0001_migrations")) run_prior_migrations(MigrationGroup.EVENTS, backfill_migration_id, runner) errors_storage = get_writable_storage(StorageKey.ERRORS) clickhouse = errors_storage.get_cluster().get_query_connection( ClickhouseClientSettings.QUERY) errors_table_name = errors_storage.get_table_writer().get_schema( ).get_table_name() raw_events = [] for i in range(10): event = get_raw_event() raw_events.append(event) events_storage = get_writable_storage(StorageKey.EVENTS) write_unprocessed_events(events_storage, raw_events) assert get_count_from_storage(errors_table_name, clickhouse) == 0 # Run 0014_backfill_errors runner.run_migration(MigrationKey(MigrationGroup.EVENTS, backfill_migration_id), force=True) assert get_count_from_storage(errors_table_name, clickhouse) == 10 outcome = perform_select_query(["contexts.key", "contexts.value"], errors_table_name, None, str(1), clickhouse) assert outcome[0] == ( [ "device.model_id", "geo.city", "geo.country_code", "geo.region", "os.kernel_version", ], ["Galaxy", "San Francisco", "US", "CA", "1.1.1"], )
def setup_method(self, test_method: Callable[..., Any]) -> None: super().setup_method(test_method) self.trace_id = uuid.UUID("7400045b-25c4-43b8-8591-4600aa83ad04") self.event = get_raw_event() self.project_id = self.event["project_id"] self.org_id = self.event["organization_id"] self.skew = timedelta(minutes=180) self.base_time = datetime.utcnow().replace( minute=0, second=0, microsecond=0) - timedelta(minutes=180) events_storage = get_entity(EntityKey.EVENTS).get_writable_storage() assert events_storage is not None write_unprocessed_events(events_storage, [self.event]) self.next_time = datetime.utcnow().replace( minute=0, second=0, microsecond=0) + timedelta(minutes=180) write_unprocessed_events( get_writable_storage(StorageKey.TRANSACTIONS), [get_raw_transaction()], )
def setup_method(self, test_method): super().setup_method(test_method) self.app.post = partial(self.app.post, headers={"referer": "test"}) self.event = get_raw_event() self.project_id = self.event["project_id"] self.base_time = datetime.utcnow().replace( second=0, microsecond=0, tzinfo=pytz.utc ) - timedelta(minutes=90) self.next_time = self.base_time + timedelta(minutes=95) self.events_storage = get_entity(EntityKey.EVENTS).get_writable_storage() write_unprocessed_events(self.events_storage, [self.event]) groups = [ { "offset": 0, "project_id": self.project_id, "id": self.event["group_id"], "record_deleted": 0, "status": 0, } ] groups_storage = get_entity(EntityKey.GROUPEDMESSAGES).get_writable_storage() groups_storage.get_table_writer().get_batch_writer( metrics=DummyMetricsBackend(strict=True) ).write([json.dumps(group).encode("utf-8") for group in groups]) assignees = [ { "offset": 0, "project_id": self.project_id, "group_id": self.event["group_id"], "record_deleted": 0, "user_id": 100, } ] assignees_storage = get_entity(EntityKey.GROUPASSIGNEE).get_writable_storage() assignees_storage.get_table_writer().get_batch_writer( metrics=DummyMetricsBackend(strict=True) ).write([json.dumps(assignee).encode("utf-8") for assignee in assignees])
def test_backfill_errors() -> None: errors_storage = get_writable_storage(StorageKey.ERRORS) clickhouse = errors_storage.get_cluster().get_query_connection( ClickhouseClientSettings.QUERY) errors_table_name = errors_storage.get_table_writer().get_schema( ).get_table_name() def get_errors_count() -> int: return clickhouse.execute( f"SELECT count() from {errors_table_name}")[0][0] raw_events = [] for i in range(10): event = get_raw_event() raw_events.append(event) events_storage = get_writable_storage(StorageKey.EVENTS) write_unprocessed_events(events_storage, raw_events) assert get_errors_count() == 0 backfill_errors() assert get_errors_count() == 10 assert clickhouse.execute( f"SELECT contexts.key, contexts.value from {errors_table_name} LIMIT 1;" )[0] == ( ( "device.model_id", "geo.city", "geo.country_code", "geo.region", "os.kernel_version", ), ("Galaxy", "San Francisco", "US", "CA", "1.1.1"), )
def setup_method(self) -> None: from snuba.web.views import application assert application.testing is True self.app = application.test_client() self.storage = get_writable_storage(StorageKey.ERRORS) self.replacer = replacer.ReplacerWorker( self.storage, CONSUMER_GROUP, DummyMetricsBackend(strict=True), ) # Total query time range is 24h before to 24h after now to account # for local machine time zones self.from_time = datetime.now().replace( minute=0, second=0, microsecond=0) - timedelta(days=1) self.to_time = self.from_time + timedelta(days=2) self.project_id = 1 self.event = get_raw_event()
process_message_multistorage, process_message_multistorage_identical_storages, ) from snuba.datasets.storages import StorageKey from tests.fixtures import get_raw_event, get_raw_transaction test_data = [ pytest.param( Message( Partition(Topic("errors"), 1), 1, MultistorageKafkaPayload( [StorageKey.ERRORS, StorageKey.ERRORS_V2], KafkaPayload( None, json.dumps((2, "insert", get_raw_event())).encode("utf-8"), [], ), ), datetime.now(), ), True, id="both errors storage", ), pytest.param( Message( Partition(Topic("errors"), 1), 1, MultistorageKafkaPayload( [StorageKey.TRANSACTIONS, StorageKey.TRANSACTIONS_V2], KafkaPayload(
def setup_method(self, test_method): super().setup_method(test_method, "events") self.metadata = KafkaMessageMetadata(0, 0, datetime.now()) self.event = get_raw_event()
def setup_method(self, test_method): self.dataset = get_dataset("events") self.metadata = KafkaMessageMetadata(0, 0, datetime.now()) self.event = get_raw_event()
def setup_method(self, test_method, dataset_name="events"): super(BaseEventsTest, self).setup_method(test_method, dataset_name) self.table = enforce_table_writer(self.dataset).get_schema().get_table_name() self.event = InsertEvent(get_raw_event())