def test_messages(self) -> None: processor = GroupAssigneeProcessor("sentry_groupasignee") metadata = KafkaMessageMetadata(offset=42, partition=0, timestamp=datetime(1970, 1, 1)) ret = processor.process_message(self.INSERT_MSG, metadata) assert ret == InsertBatch([self.PROCESSED]) self.write_processed_messages([ret]) ret = (get_cluster(StorageSetKey.EVENTS).get_query_connection( ClickhouseClientSettings.QUERY).execute( "SELECT * FROM groupassignee_local;")) assert ret[0] == ( 42, # offset 0, # deleted 2, # project_id 1359, # group_id datetime(2019, 9, 19, 0, 17, 55), 1, # user_id None, # team_id ) ret = processor.process_message(self.UPDATE_MSG_NO_KEY_CHANGE, metadata) assert ret == InsertBatch([self.PROCESSED]) # Tests an update with key change which becomes a two inserts: # one deletion and the insertion of the new row. ret = processor.process_message(self.UPDATE_MSG_WITH_KEY_CHANGE, metadata) assert ret == InsertBatch([self.DELETED, self.PROCESSED_UPDATE]) ret = processor.process_message(self.DELETE_MSG, metadata) assert ret == InsertBatch([self.DELETED])
def test_messages(self): processor = GroupAssigneeProcessor('sentry_groupasignee') metadata = KafkaMessageMetadata( offset=42, partition=0, ) begin_msg = json.loads(self.BEGIN_MSG) ret = processor.process_message(begin_msg, metadata) assert ret is None commit_msg = json.loads(self.COMMIT_MSG) ret = processor.process_message(commit_msg, metadata) assert ret is None insert_msg = json.loads(self.INSERT_MSG) ret = processor.process_message(insert_msg, metadata) assert ret.data == [self.PROCESSED] self.write_processed_records(ret.data) cp = ClickhousePool() ret = cp.execute("SELECT * FROM test_groupassignee_local;") assert ret[0] == ( 42, # offset 0, # deleted 2, # project_id 1359, # group_id datetime(2019, 9, 19, 0, 17, 55), 1, # user_id None, # team_id ) update_msg = json.loads(self.UPDATE_MSG_NO_KEY_CHANGE) ret = processor.process_message(update_msg, metadata) assert ret.data == [self.PROCESSED] # Tests an update with key change which becomes a two inserts: # one deletion and the insertion of the new row. update_msg = json.loads(self.UPDATE_MSG_WITH_KEY_CHANGE) ret = processor.process_message(update_msg, metadata) assert ret.data == [self.DELETED, self.PROCESSED_UPDATE] delete_msg = json.loads(self.DELETE_MSG) ret = processor.process_message(delete_msg, metadata) assert ret.data == [self.DELETED]
def __init__(self) -> None: columns = ColumnSet([ # columns to maintain the dataset # Kafka topic offset ("offset", UInt(64)), ("record_deleted", UInt(8)), # PG columns ("project_id", UInt(64)), ("group_id", UInt(64)), ("date_added", Nullable(DateTime())), ("user_id", Nullable(UInt(64))), ("team_id", Nullable(UInt(64))), ]) schema = ReplacingMergeTreeSchema( columns=columns, local_table_name='groupassignee_local', dist_table_name='groupassignee_dist', order_by='(project_id, group_id)', partition_by=None, version_column='offset', ) dataset_schemas = DatasetSchemas( read_schema=schema, write_schema=schema, ) super().__init__( dataset_schemas=dataset_schemas, table_writer=GroupAssigneeTableWriter( write_schema=schema, stream_loader=KafkaStreamLoader( processor=GroupAssigneeProcessor(self.POSTGRES_TABLE), default_topic="cdc", ), postgres_table=self.POSTGRES_TABLE, ), default_control_topic="cdc_control", postgres_table=self.POSTGRES_TABLE, )
("user_id", UInt(64, Modifiers(nullable=True))), ("team_id", UInt(64, Modifiers(nullable=True))), ]) schema = WritableTableSchema( columns=columns, local_table_name="groupassignee_local", dist_table_name="groupassignee_dist", storage_set_key=StorageSetKey.CDC, ) POSTGRES_TABLE = "sentry_groupasignee" storage = CdcStorage( storage_key=StorageKey.GROUPASSIGNEES, storage_set_key=StorageSetKey.CDC, schema=schema, query_processors=[ PrewhereProcessor(["project_id"]), ConsistencyEnforcerProcessor(), ], stream_loader=build_kafka_stream_loader_from_settings( processor=GroupAssigneeProcessor(POSTGRES_TABLE), default_topic=Topic.CDC, pre_filter=CdcTableNameMessageFilter(POSTGRES_TABLE), ), default_control_topic="cdc_control", postgres_table=POSTGRES_TABLE, row_processor=lambda row: GroupAssigneeRow.from_bulk(row).to_clickhouse(), )
def test_messages(self): processor = GroupAssigneeProcessor("sentry_groupasignee") message_filter = CdcTableNameMessageFilter( postgres_table=POSTGRES_TABLE) metadata = KafkaMessageMetadata(offset=42, partition=0, timestamp=datetime(1970, 1, 1)) assert message_filter.should_drop( self.__make_msg(0, 42, self.BEGIN_MSG, [])) assert message_filter.should_drop( self.__make_msg(0, 42, self.COMMIT_MSG, [])) assert not message_filter.should_drop( self.__make_msg(0, 42, self.INSERT_MSG, [("table", POSTGRES_TABLE.encode())])) insert_msg = json.loads(self.INSERT_MSG) ret = processor.process_message(insert_msg, metadata) assert ret == InsertBatch([self.PROCESSED]) self.write_processed_messages([ret]) ret = (get_cluster(StorageSetKey.EVENTS).get_query_connection( ClickhouseClientSettings.QUERY).execute( "SELECT * FROM groupassignee_local;")) assert ret[0] == ( 42, # offset 0, # deleted 2, # project_id 1359, # group_id datetime(2019, 9, 19, 0, 17, 55), 1, # user_id None, # team_id ) assert not message_filter.should_drop( self.__make_msg( 0, 42, self.UPDATE_MSG_NO_KEY_CHANGE, [("table", POSTGRES_TABLE.encode())], )) update_msg = json.loads(self.UPDATE_MSG_NO_KEY_CHANGE) ret = processor.process_message(update_msg, metadata) assert ret == InsertBatch([self.PROCESSED]) # Tests an update with key change which becomes a two inserts: # one deletion and the insertion of the new row. assert not message_filter.should_drop( self.__make_msg( 0, 42, self.UPDATE_MSG_WITH_KEY_CHANGE, [("table", POSTGRES_TABLE.encode())], )) update_msg = json.loads(self.UPDATE_MSG_WITH_KEY_CHANGE) ret = processor.process_message(update_msg, metadata) assert ret == InsertBatch([self.DELETED, self.PROCESSED_UPDATE]) assert not message_filter.should_drop( self.__make_msg( 0, 42, self.DELETE_MSG, [("table", POSTGRES_TABLE.encode())], )) delete_msg = json.loads(self.DELETE_MSG) ret = processor.process_message(delete_msg, metadata) assert ret == InsertBatch([self.DELETED])
def test_messages(self): processor = GroupAssigneeProcessor("sentry_groupasignee") message_filter = CdcTableNameMessageFilter( postgres_table=POSTGRES_TABLE) metadata = KafkaMessageMetadata( offset=42, partition=0, ) assert not message_filter.should_drop( self.__make_msg(0, 42, self.BEGIN_MSG, [])) begin_msg = json.loads(self.BEGIN_MSG) ret = processor.process_message(begin_msg, metadata) assert ret is None assert not message_filter.should_drop( self.__make_msg(0, 42, self.COMMIT_MSG, [])) commit_msg = json.loads(self.COMMIT_MSG) ret = processor.process_message(commit_msg, metadata) assert ret is None assert not message_filter.should_drop( self.__make_msg(0, 42, self.INSERT_MSG, [("table", POSTGRES_TABLE.encode())])) insert_msg = json.loads(self.INSERT_MSG) ret = processor.process_message(insert_msg, metadata) assert ret.data == [self.PROCESSED] self.write_processed_records(ret.data) ret = clickhouse_ro.execute("SELECT * FROM test_groupassignee_local;") assert ret[0] == ( 42, # offset 0, # deleted 2, # project_id 1359, # group_id datetime(2019, 9, 19, 0, 17, 55), 1, # user_id None, # team_id ) assert not message_filter.should_drop( self.__make_msg( 0, 42, self.UPDATE_MSG_NO_KEY_CHANGE, [("table", POSTGRES_TABLE.encode())], )) update_msg = json.loads(self.UPDATE_MSG_NO_KEY_CHANGE) ret = processor.process_message(update_msg, metadata) assert ret.data == [self.PROCESSED] # Tests an update with key change which becomes a two inserts: # one deletion and the insertion of the new row. assert not message_filter.should_drop( self.__make_msg( 0, 42, self.UPDATE_MSG_WITH_KEY_CHANGE, [("table", POSTGRES_TABLE.encode())], )) update_msg = json.loads(self.UPDATE_MSG_WITH_KEY_CHANGE) ret = processor.process_message(update_msg, metadata) assert ret.data == [self.DELETED, self.PROCESSED_UPDATE] assert not message_filter.should_drop( self.__make_msg(0, 42, self.DELETE_MSG, [])) delete_msg = json.loads(self.DELETE_MSG) ret = processor.process_message(delete_msg, metadata) assert ret.data == [self.DELETED]