def test_hash_invalid_primary_hash(self): self.event[ 'primary_hash'] = b"'tinymce' \u063a\u064a\u0631 \u0645\u062d".decode( 'unicode-escape') _, processed = process_message(self.event) assert processed['primary_hash'] == 'a52ccc1a61c2258e918b43b5aff50db1'
def test_columns_match_schema(self): _, processed = process_message(self.event) row = row_from_processed_event(processed) # verify that the 'count of columns from event' + 'count of columns from metadata' # equals the 'count of columns' in the processed row tuple # note that the content is verified in processor tests assert (len(processed) + len(METADATA_COLUMNS)) == len(row)
def write_raw_events(self, events): if not isinstance(events, (list, tuple)): events = [events] out = [] for event in events: if 'primary_hash' not in event: event = self.wrap_raw_event(event) _, processed = process_message(event) out.append(processed) return self.write_processed_events(out)
def test_unknown_columns(self): """Fields in a processed events are ignored if they don't have a corresponding Clickhouse column declared.""" _, processed = process_message(self.event) assert 'sdk_name' in processed sdk_name = processed['sdk_name'] columns_copy = ColumnSet( [col for col in ALL_COLUMNS.columns if not col.name == 'sdk_name']) assert len(columns_copy) == (len(ALL_COLUMNS) - 1) row = row_from_processed_event(processed, columns_copy) assert len(row) == len(columns_copy) assert sdk_name not in row
def write(): from snuba.processor import process_message from snuba.writer import row_from_processed_event, write_rows body = json.loads(request.data) rows = [] for event in body: _, processed = process_message(event) row = row_from_processed_event(processed) rows.append(row) ensure_table_exists() write_rows( clickhouse_rw, table=settings.CLICKHOUSE_TABLE, rows=rows ) return ('ok', 200, {'Content-Type': 'text/plain'})
def test_v2_end_delete_tag(self): project_id = 1 message = (2, 'end_delete_tag', {'project_id': project_id}) assert processor.process_message(message) == \ (processor.REPLACE, (six.text_type(project_id), message))
def test_simple(self): _, processed = process_message(self.event) for field in ('event_id', 'project_id', 'message', 'platform'): assert processed[field] == self.event[field]
def test_v1_delete_groups_skipped(self): assert processor.process_message((1, 'delete_groups', {})) is None
def test_unexpected_obj(self): self.event['message'] = {'what': 'why is this in the message'} _, processed = process_message(self.event) assert processed['message'] == '{"what": "why is this in the message"}'
def test_invalid_format(self): with pytest.raises(InvalidMessageVersion): process_message((-1, 'insert', self.event))
def test_invalid_version(self): with pytest.raises(InvalidMessageVersion): process_message((2**32 - 1, 'insert', self.event))
def test_invalid_type_version_0(self): with pytest.raises(InvalidMessageType): process_message((0, 'invalid', self.event))
def test_simple_version_1(self): assert process_message((0, 'insert', self.event)) == process_message( (1, 'insert', self.event, {}))
def test_v2_start_unmerge(self): project_id = 1 message = (2, 'start_unmerge', {'project_id': project_id}) assert processor.process_message(message) == \ (processor.REPLACE, (six.text_type(project_id), message))
def test_simple_version_0(self): _, processed = process_message((0, 'insert', self.event)) for field in ('event_id', 'project_id', 'message', 'platform'): assert processed[field] == self.event[field]
def test_v2_invalid_type(self): with pytest.raises(processor.InvalidMessageType): assert processor.process_message((2, '__invalid__', {})) == 1
def test_v1_unmerge_skipped(self): assert processor.process_message((1, 'unmerge', {})) is None