def extract_lines_from_request(request): # type: (Union[AddEventsRequest, List[AddEventsRequest]]) -> Union[List[six.text_type],Set[six.text_type]] """Extract lines from AddEventsRequest""" result = list() # if we test ShardedCopyingManager, it returns multiple requests for each of its worker sessions, if isinstance(request, (list, tuple)): for req in request: lines = extract_lines_from_request(req) result.append(lines) return list(itertools.chain(*result)) # extract single request. parsed_request = test_util.parse_scalyr_request(request.get_payload()) lines = [] if "events" in parsed_request: for event in parsed_request["events"]: if "attrs" in event: attrs = event["attrs"] if "message" in attrs: lines.append(attrs["message"].strip()) return lines
def test_same_sequence_id(self): expected_id = b"1234" expected_number = 1234 expected_delta = 1 request = AddEventsRequest(self.__body) request.set_client_time(1) self.assertTrue( request.add_event( Event().set_message("eventOne"), timestamp=1, sequence_id=expected_id, sequence_number=expected_number, )) self.assertTrue( request.add_event( Event().set_message("eventTwo"), timestamp=2, sequence_id=expected_id, sequence_number=expected_number + expected_delta, )) self.assertEquals(request.total_events, 2) json = test_util.parse_scalyr_request(request.get_payload()) event = json["events"][1] self.assertFalse("si" in event) self.assertFalse("sn" in event) self.assertEquals(expected_delta, event["sd"]) request.close()
def test_all_fields(self): x = Event(thread_id="foo", attrs={"parser": "bar"}) x.set_message(b"my_message") x.set_sampling_rate(0.5) x.set_sequence_id(1) x.set_sequence_number(2) x.set_sequence_number_delta(3) x.set_timestamp(42) output_buffer = BytesIO() x.serialize(output_buffer) self.assertEquals( b'{thread:"foo", log:"foo", attrs:{"parser":"bar",message:`s\x00\x00\x00\nmy_message,sample_rate:0.5},ts:"42",si:"1",sn:2,sd:3}', output_buffer.getvalue(), ) self.assertEquals( { "log": "foo", "thread": "foo", "ts": "42", "si": "1", "sn": 2, "sd": 3, "attrs": { "parser": "bar", "message": "my_message", "sample_rate": 0.5 }, }, test_util.parse_scalyr_request(output_buffer.getvalue()), )
def test_send_request_timestamp_increases_monotonically(self): session = ScalyrClientSession( "https://dummserver.com", "DUMMY API KEY", SCALYR_VERSION, enforce_monotonic_timestamps=True, ) session._ScalyrClientSession__connection = mock.Mock() session._ScalyrClientSession__receive_response = mock.Mock() scalyr_client._set_last_timestamp(0) add_events_request = session.add_events_request() ts = 2000 expected = str(ts + 1) add_events_request.add_event(Event().set_message("eventOne"), timestamp=ts) add_events_request.add_event(Event().set_message("eventTwo"), timestamp=1) json = test_util.parse_scalyr_request(add_events_request.get_payload()) event = json["events"][1] self.assertEquals(event["ts"], expected)
def test_no_monotonically_increasing_timestamp(self): request = AddEventsRequest(self.__body) ts = 2000 self.assertTrue( request.add_event(Event().set_message("eventOne"), timestamp=ts)) self.assertTrue( request.add_event(Event().set_message("eventTwo"), timestamp=1)) json = test_util.parse_scalyr_request(request.get_payload()) event = json["events"][1] self.assertEquals(event["ts"], "1")
def test_sequence_number_but_no_id(self): request = AddEventsRequest(self.__body) request.set_client_time(1) self.assertTrue( request.add_event(Event().set_message("eventOne"), timestamp=1, sequence_number=1234)) self.assertEquals(request.total_events, 1) json = test_util.parse_scalyr_request(request.get_payload()) event = json["events"][0] self.assertFalse("si" in event) self.assertFalse("sn" in event) self.assertFalse("sd" in event) request.close()
def test_timestamp_none(self): request = AddEventsRequest(self.__body) request.set_client_time(100) ts = int(time.time() * 1e9) self.assertTrue( request.add_event(Event().set_message("eventOne"), timestamp=None)) json = test_util.parse_scalyr_request(request.get_payload()) event = json["events"][0] event_ts = int(event["ts"]) threshold = abs(event_ts - ts) # allow a threshold of 1 second to have elapsed between reading the time.time and # setting the event timestamp in add_event self.assertTrue(threshold < 1e9)
def test_monotonically_increasing_timestamp(self): request = AddEventsRequest(self.__body, enforce_monotonic_timestamps=True) scalyr_client._set_last_timestamp(0) ts = 2000 expected = str(ts + 1) self.assertTrue( request.add_event(Event().set_message("eventOne"), timestamp=ts)) self.assertTrue( request.add_event(Event().set_message("eventTwo"), timestamp=1)) json = test_util.parse_scalyr_request(request.get_payload()) event = json["events"][1] self.assertEquals(event["ts"], expected)
def test_exceeds_size_doesnt_effect_sequence(self): first_id = "1234" second_id = "1235" first_number = 1234 second_number = 4321 expected_delta = 10 request = AddEventsRequest(self.__body, max_size=180) request.set_client_time(1) self.assertTrue( request.add_event( Event().set_message("eventOne"), timestamp=1, sequence_id=first_id, sequence_number=first_number, ) ) self.assertFalse( request.add_event( Event( attrs={"name": "eventTwo", "long": "some really long text"} ).set_message("eventTwo"), timestamp=2, sequence_id=second_id, sequence_number=second_number, ) ) self.assertTrue( request.add_event( Event().set_message("eventThree"), timestamp=3, sequence_id=first_id, sequence_number=first_number + expected_delta, ) ) self.assertEquals(request.total_events, 2) json = test_util.parse_scalyr_request(request.get_payload()) event = json["events"][1] self.assertFalse("si" in event) self.assertEquals(expected_delta, event["sd"]) self.assertFalse("sn" in event) request.close()
def test_set_position_resets_sequence_compression(self): first_id = "1234" second_id = "1235" first_number = 1234 second_number = 4321 expected_delta = 10 request = AddEventsRequest(self.__body) request.set_client_time(1) self.assertTrue( request.add_event( Event().set_message("eventOne"), timestamp=1, sequence_id=first_id, sequence_number=first_number, ) ) position = request.position() self.assertTrue( request.add_event( Event().set_message("eventTwo"), timestamp=2, sequence_id=first_id, sequence_number=first_number + expected_delta, ) ) request.set_position(position) self.assertTrue( request.add_event( Event().set_message("eventThree"), timestamp=3, sequence_id=first_id, sequence_number=second_number, ) ) self.assertEquals(request.total_events, 2) json = test_util.parse_scalyr_request(request.get_payload()) event = json["events"][1] self.assertEquals(second_number, event["sn"]) self.assertFalse("sd" in event) request.close()
def test_sequence_id_and_number(self): expected_id = "1234" expected_number = 1234 request = AddEventsRequest(self.__body) request.set_client_time(1) self.assertTrue( request.add_event( Event().set_message("eventOne"), timestamp=1, sequence_id=expected_id, sequence_number=expected_number, )) self.assertEquals(request.total_events, 1) json = test_util.parse_scalyr_request(request.get_payload()) event = json["events"][0] self.assertEquals(expected_id, event["si"]) self.assertEquals(expected_number, event["sn"]) self.assertFalse("sd" in event) request.close()
def test_different_sequence_id(self): first_id = "1234" second_id = "1235" first_number = 1234 second_number = 1234 request = AddEventsRequest(self.__body) request.set_client_time(1) self.assertTrue( request.add_event( Event().set_message("eventOne"), timestamp=1, sequence_id=first_id, sequence_number=first_number, ) ) self.assertTrue( request.add_event( Event().set_message("eventTwo"), timestamp=2, sequence_id=first_id, sequence_number=first_number + 1, ) ) self.assertTrue( request.add_event( Event().set_message("eventThree"), timestamp=3, sequence_id=second_id, sequence_number=second_number, ) ) self.assertEquals(request.total_events, 3) json = test_util.parse_scalyr_request(request.get_payload()) event = json["events"][2] self.assertEquals(second_id, event["si"]) self.assertEquals(second_number, event["sn"]) self.assertFalse("sd" in event) request.close()