def extract_lines_from_request(request):
# type: (Union[AddEventsRequest, List[AddEventsRequest]]) -> Union[List[six.text_type],Set[six.text_type]]
"""Extract lines from AddEventsRequest"""
result = list()
# if we test ShardedCopyingManager, it returns multiple requests for each of its worker sessions,
if isinstance(request, (list, tuple)):
for req in request:
lines = extract_lines_from_request(req)
result.append(lines)
return list(itertools.chain(*result))
# extract single request.
parsed_request = test_util.parse_scalyr_request(request.get_payload())
lines = []
if "events" in parsed_request:
for event in parsed_request["events"]:
if "attrs" in event:
attrs = event["attrs"]
if "message" in attrs:
lines.append(attrs["message"].strip())
return lines
def test_same_sequence_id(self):
expected_id = b"1234"
expected_number = 1234
expected_delta = 1
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertTrue(
request.add_event(
Event().set_message("eventOne"),
timestamp=1,
sequence_id=expected_id,
sequence_number=expected_number,
))
self.assertTrue(
request.add_event(
Event().set_message("eventTwo"),
timestamp=2,
sequence_id=expected_id,
sequence_number=expected_number + expected_delta,
))
self.assertEquals(request.total_events, 2)
json = test_util.parse_scalyr_request(request.get_payload())
event = json["events"][1]
self.assertFalse("si" in event)
self.assertFalse("sn" in event)
self.assertEquals(expected_delta, event["sd"])
request.close()
def test_all_fields(self):
x = Event(thread_id="foo", attrs={"parser": "bar"})
x.set_message(b"my_message")
x.set_sampling_rate(0.5)
x.set_sequence_id(1)
x.set_sequence_number(2)
x.set_sequence_number_delta(3)
x.set_timestamp(42)
output_buffer = BytesIO()
x.serialize(output_buffer)
self.assertEquals(
b'{thread:"foo", log:"foo", attrs:{"parser":"bar",message:`s\x00\x00\x00\nmy_message,sample_rate:0.5},ts:"42",si:"1",sn:2,sd:3}',
output_buffer.getvalue(),
)
self.assertEquals(
{
"log": "foo",
"thread": "foo",
"ts": "42",
"si": "1",
"sn": 2,
"sd": 3,
"attrs": {
"parser": "bar",
"message": "my_message",
"sample_rate": 0.5
},
},
test_util.parse_scalyr_request(output_buffer.getvalue()),
)
def test_send_request_timestamp_increases_monotonically(self):
session = ScalyrClientSession(
"https://dummserver.com",
"DUMMY API KEY",
SCALYR_VERSION,
enforce_monotonic_timestamps=True,
)
session._ScalyrClientSession__connection = mock.Mock()
session._ScalyrClientSession__receive_response = mock.Mock()
scalyr_client._set_last_timestamp(0)
add_events_request = session.add_events_request()
ts = 2000
expected = str(ts + 1)
add_events_request.add_event(Event().set_message("eventOne"),
timestamp=ts)
add_events_request.add_event(Event().set_message("eventTwo"),
timestamp=1)
json = test_util.parse_scalyr_request(add_events_request.get_payload())
event = json["events"][1]
self.assertEquals(event["ts"], expected)
def test_no_monotonically_increasing_timestamp(self):
request = AddEventsRequest(self.__body)
ts = 2000
self.assertTrue(
request.add_event(Event().set_message("eventOne"), timestamp=ts))
self.assertTrue(
request.add_event(Event().set_message("eventTwo"), timestamp=1))
json = test_util.parse_scalyr_request(request.get_payload())
event = json["events"][1]
self.assertEquals(event["ts"], "1")
def test_sequence_number_but_no_id(self):
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertTrue(
request.add_event(Event().set_message("eventOne"),
timestamp=1,
sequence_number=1234))
self.assertEquals(request.total_events, 1)
json = test_util.parse_scalyr_request(request.get_payload())
event = json["events"][0]
self.assertFalse("si" in event)
self.assertFalse("sn" in event)
self.assertFalse("sd" in event)
request.close()
def test_timestamp_none(self):
request = AddEventsRequest(self.__body)
request.set_client_time(100)
ts = int(time.time() * 1e9)
self.assertTrue(
request.add_event(Event().set_message("eventOne"), timestamp=None))
json = test_util.parse_scalyr_request(request.get_payload())
event = json["events"][0]
event_ts = int(event["ts"])
threshold = abs(event_ts - ts)
# allow a threshold of 1 second to have elapsed between reading the time.time and
# setting the event timestamp in add_event
self.assertTrue(threshold < 1e9)
def test_monotonically_increasing_timestamp(self):
request = AddEventsRequest(self.__body,
enforce_monotonic_timestamps=True)
scalyr_client._set_last_timestamp(0)
ts = 2000
expected = str(ts + 1)
self.assertTrue(
request.add_event(Event().set_message("eventOne"), timestamp=ts))
self.assertTrue(
request.add_event(Event().set_message("eventTwo"), timestamp=1))
json = test_util.parse_scalyr_request(request.get_payload())
event = json["events"][1]
self.assertEquals(event["ts"], expected)
def test_exceeds_size_doesnt_effect_sequence(self):
first_id = "1234"
second_id = "1235"
first_number = 1234
second_number = 4321
expected_delta = 10
request = AddEventsRequest(self.__body, max_size=180)
request.set_client_time(1)
self.assertTrue(
request.add_event(
Event().set_message("eventOne"),
timestamp=1,
sequence_id=first_id,
sequence_number=first_number,
)
)
self.assertFalse(
request.add_event(
Event(
attrs={"name": "eventTwo", "long": "some really long text"}
).set_message("eventTwo"),
timestamp=2,
sequence_id=second_id,
sequence_number=second_number,
)
)
self.assertTrue(
request.add_event(
Event().set_message("eventThree"),
timestamp=3,
sequence_id=first_id,
sequence_number=first_number + expected_delta,
)
)
self.assertEquals(request.total_events, 2)
json = test_util.parse_scalyr_request(request.get_payload())
event = json["events"][1]
self.assertFalse("si" in event)
self.assertEquals(expected_delta, event["sd"])
self.assertFalse("sn" in event)
request.close()
def test_set_position_resets_sequence_compression(self):
first_id = "1234"
second_id = "1235"
first_number = 1234
second_number = 4321
expected_delta = 10
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertTrue(
request.add_event(
Event().set_message("eventOne"),
timestamp=1,
sequence_id=first_id,
sequence_number=first_number,
)
)
position = request.position()
self.assertTrue(
request.add_event(
Event().set_message("eventTwo"),
timestamp=2,
sequence_id=first_id,
sequence_number=first_number + expected_delta,
)
)
request.set_position(position)
self.assertTrue(
request.add_event(
Event().set_message("eventThree"),
timestamp=3,
sequence_id=first_id,
sequence_number=second_number,
)
)
self.assertEquals(request.total_events, 2)
json = test_util.parse_scalyr_request(request.get_payload())
event = json["events"][1]
self.assertEquals(second_number, event["sn"])
self.assertFalse("sd" in event)
request.close()
def test_sequence_id_and_number(self):
expected_id = "1234"
expected_number = 1234
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertTrue(
request.add_event(
Event().set_message("eventOne"),
timestamp=1,
sequence_id=expected_id,
sequence_number=expected_number,
))
self.assertEquals(request.total_events, 1)
json = test_util.parse_scalyr_request(request.get_payload())
event = json["events"][0]
self.assertEquals(expected_id, event["si"])
self.assertEquals(expected_number, event["sn"])
self.assertFalse("sd" in event)
request.close()
def test_different_sequence_id(self):
first_id = "1234"
second_id = "1235"
first_number = 1234
second_number = 1234
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertTrue(
request.add_event(
Event().set_message("eventOne"),
timestamp=1,
sequence_id=first_id,
sequence_number=first_number,
)
)
self.assertTrue(
request.add_event(
Event().set_message("eventTwo"),
timestamp=2,
sequence_id=first_id,
sequence_number=first_number + 1,
)
)
self.assertTrue(
request.add_event(
Event().set_message("eventThree"),
timestamp=3,
sequence_id=second_id,
sequence_number=second_number,
)
)
self.assertEquals(request.total_events, 3)
json = test_util.parse_scalyr_request(request.get_payload())
event = json["events"][2]
self.assertEquals(second_id, event["si"])
self.assertEquals(second_number, event["sn"])
self.assertFalse("sd" in event)
request.close()
