def test_messages(self) -> None: processor = GroupAssigneeProcessor("sentry_groupasignee") metadata = KafkaMessageMetadata(offset=42, partition=0, timestamp=datetime(1970, 1, 1)) ret = processor.process_message(self.INSERT_MSG, metadata) assert ret == InsertBatch([self.PROCESSED]) self.write_processed_messages([ret]) ret = (get_cluster(StorageSetKey.EVENTS).get_query_connection( ClickhouseClientSettings.QUERY).execute( "SELECT * FROM groupassignee_local;")) assert ret[0] == ( 42, # offset 0, # deleted 2, # project_id 1359, # group_id datetime(2019, 9, 19, 0, 17, 55), 1, # user_id None, # team_id ) ret = processor.process_message(self.UPDATE_MSG_NO_KEY_CHANGE, metadata) assert ret == InsertBatch([self.PROCESSED]) # Tests an update with key change which becomes a two inserts: # one deletion and the insertion of the new row. ret = processor.process_message(self.UPDATE_MSG_WITH_KEY_CHANGE, metadata) assert ret == InsertBatch([self.DELETED, self.PROCESSED_UPDATE]) ret = processor.process_message(self.DELETE_MSG, metadata) assert ret == InsertBatch([self.DELETED])
def test_messages(self): processor = GroupAssigneeProcessor('sentry_groupasignee') metadata = KafkaMessageMetadata( offset=42, partition=0, ) begin_msg = json.loads(self.BEGIN_MSG) ret = processor.process_message(begin_msg, metadata) assert ret is None commit_msg = json.loads(self.COMMIT_MSG) ret = processor.process_message(commit_msg, metadata) assert ret is None insert_msg = json.loads(self.INSERT_MSG) ret = processor.process_message(insert_msg, metadata) assert ret.data == [self.PROCESSED] self.write_processed_records(ret.data) cp = ClickhousePool() ret = cp.execute("SELECT * FROM test_groupassignee_local;") assert ret[0] == ( 42, # offset 0, # deleted 2, # project_id 1359, # group_id datetime(2019, 9, 19, 0, 17, 55), 1, # user_id None, # team_id ) update_msg = json.loads(self.UPDATE_MSG_NO_KEY_CHANGE) ret = processor.process_message(update_msg, metadata) assert ret.data == [self.PROCESSED] # Tests an update with key change which becomes a two inserts: # one deletion and the insertion of the new row. update_msg = json.loads(self.UPDATE_MSG_WITH_KEY_CHANGE) ret = processor.process_message(update_msg, metadata) assert ret.data == [self.DELETED, self.PROCESSED_UPDATE] delete_msg = json.loads(self.DELETE_MSG) ret = processor.process_message(delete_msg, metadata) assert ret.data == [self.DELETED]
def test_messages(self): processor = GroupAssigneeProcessor("sentry_groupasignee") message_filter = CdcTableNameMessageFilter( postgres_table=POSTGRES_TABLE) metadata = KafkaMessageMetadata(offset=42, partition=0, timestamp=datetime(1970, 1, 1)) assert message_filter.should_drop( self.__make_msg(0, 42, self.BEGIN_MSG, [])) assert message_filter.should_drop( self.__make_msg(0, 42, self.COMMIT_MSG, [])) assert not message_filter.should_drop( self.__make_msg(0, 42, self.INSERT_MSG, [("table", POSTGRES_TABLE.encode())])) insert_msg = json.loads(self.INSERT_MSG) ret = processor.process_message(insert_msg, metadata) assert ret == InsertBatch([self.PROCESSED]) self.write_processed_messages([ret]) ret = (get_cluster(StorageSetKey.EVENTS).get_query_connection( ClickhouseClientSettings.QUERY).execute( "SELECT * FROM groupassignee_local;")) assert ret[0] == ( 42, # offset 0, # deleted 2, # project_id 1359, # group_id datetime(2019, 9, 19, 0, 17, 55), 1, # user_id None, # team_id ) assert not message_filter.should_drop( self.__make_msg( 0, 42, self.UPDATE_MSG_NO_KEY_CHANGE, [("table", POSTGRES_TABLE.encode())], )) update_msg = json.loads(self.UPDATE_MSG_NO_KEY_CHANGE) ret = processor.process_message(update_msg, metadata) assert ret == InsertBatch([self.PROCESSED]) # Tests an update with key change which becomes a two inserts: # one deletion and the insertion of the new row. assert not message_filter.should_drop( self.__make_msg( 0, 42, self.UPDATE_MSG_WITH_KEY_CHANGE, [("table", POSTGRES_TABLE.encode())], )) update_msg = json.loads(self.UPDATE_MSG_WITH_KEY_CHANGE) ret = processor.process_message(update_msg, metadata) assert ret == InsertBatch([self.DELETED, self.PROCESSED_UPDATE]) assert not message_filter.should_drop( self.__make_msg( 0, 42, self.DELETE_MSG, [("table", POSTGRES_TABLE.encode())], )) delete_msg = json.loads(self.DELETE_MSG) ret = processor.process_message(delete_msg, metadata) assert ret == InsertBatch([self.DELETED])
def test_messages(self): processor = GroupAssigneeProcessor("sentry_groupasignee") message_filter = CdcTableNameMessageFilter( postgres_table=POSTGRES_TABLE) metadata = KafkaMessageMetadata( offset=42, partition=0, ) assert not message_filter.should_drop( self.__make_msg(0, 42, self.BEGIN_MSG, [])) begin_msg = json.loads(self.BEGIN_MSG) ret = processor.process_message(begin_msg, metadata) assert ret is None assert not message_filter.should_drop( self.__make_msg(0, 42, self.COMMIT_MSG, [])) commit_msg = json.loads(self.COMMIT_MSG) ret = processor.process_message(commit_msg, metadata) assert ret is None assert not message_filter.should_drop( self.__make_msg(0, 42, self.INSERT_MSG, [("table", POSTGRES_TABLE.encode())])) insert_msg = json.loads(self.INSERT_MSG) ret = processor.process_message(insert_msg, metadata) assert ret.data == [self.PROCESSED] self.write_processed_records(ret.data) ret = clickhouse_ro.execute("SELECT * FROM test_groupassignee_local;") assert ret[0] == ( 42, # offset 0, # deleted 2, # project_id 1359, # group_id datetime(2019, 9, 19, 0, 17, 55), 1, # user_id None, # team_id ) assert not message_filter.should_drop( self.__make_msg( 0, 42, self.UPDATE_MSG_NO_KEY_CHANGE, [("table", POSTGRES_TABLE.encode())], )) update_msg = json.loads(self.UPDATE_MSG_NO_KEY_CHANGE) ret = processor.process_message(update_msg, metadata) assert ret.data == [self.PROCESSED] # Tests an update with key change which becomes a two inserts: # one deletion and the insertion of the new row. assert not message_filter.should_drop( self.__make_msg( 0, 42, self.UPDATE_MSG_WITH_KEY_CHANGE, [("table", POSTGRES_TABLE.encode())], )) update_msg = json.loads(self.UPDATE_MSG_WITH_KEY_CHANGE) ret = processor.process_message(update_msg, metadata) assert ret.data == [self.DELETED, self.PROCESSED_UPDATE] assert not message_filter.should_drop( self.__make_msg(0, 42, self.DELETE_MSG, [])) delete_msg = json.loads(self.DELETE_MSG) ret = processor.process_message(delete_msg, metadata) assert ret.data == [self.DELETED]