def test_set_log_line_attributes_with_base_attributes(self):
request = AddEventsRequest(self.__body)
request.set_client_time(1)
request.add_log_and_thread("log2", "Log two", {})
event_base = Event()
event_base.add_attributes({
"source": "stdout",
"base": "base"
},
overwrite_existing=False)
event_one = Event(base=event_base)
event_one.set_message("eventOne")
event_one.add_attributes({
"source": "stdin",
"event": "event"
},
overwrite_existing=True)
self.assertTrue(request.add_event(event_one, timestamp=1))
self.assertEquals(
request.get_payload(),
b"""{"token":"fakeToken", events: [{attrs:{"event":"event","source":"stdin",message:`s\x00\x00\x00\x08eventOne},ts:"1"}], """
b"""logs: [{"attrs":{},"id":"log2"}], threads: [{"id":"log2","name":"Log two"}], client_time: 1 }""",
)
request.close()
def test_same_sequence_id(self):
expected_id = b"1234"
expected_number = 1234
expected_delta = 1
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertTrue(
request.add_event(
Event().set_message("eventOne"),
timestamp=1,
sequence_id=expected_id,
sequence_number=expected_number,
))
self.assertTrue(
request.add_event(
Event().set_message("eventTwo"),
timestamp=2,
sequence_id=expected_id,
sequence_number=expected_number + expected_delta,
))
self.assertEquals(request.total_events, 2)
json = test_util.parse_scalyr_request(request.get_payload())
event = json["events"][1]
self.assertFalse("si" in event)
self.assertFalse("sn" in event)
self.assertEquals(expected_delta, event["sd"])
request.close()
def test_memento(self):
first_id = "1234"
second_id = "1235"
first_number = 1234
second_number = 1234
event = Event()
self.event_sequencer.add_sequence_fields(event, first_id, first_number)
memento = self.event_sequencer.get_memento()
event = Event()
self.event_sequencer.add_sequence_fields(event, second_id,
second_number)
self.assertIsNotNone(event.sequence_id)
self.assertIsNotNone(event.sequence_number)
self.assertIsNone(event.sequence_number_delta)
self.event_sequencer.restore_from_memento(memento)
event = Event()
self.event_sequencer.add_sequence_fields(event, first_id,
first_number + 1)
self.assertIsNone(event.sequence_id)
self.assertIsNone(event.sequence_number)
self.assertIsNotNone(event.sequence_number_delta)
def test_send_request_timestamp_increases_monotonically(self):
session = ScalyrClientSession(
"https://dummserver.com",
"DUMMY API KEY",
SCALYR_VERSION,
enforce_monotonic_timestamps=True,
)
session._ScalyrClientSession__connection = mock.Mock()
session._ScalyrClientSession__receive_response = mock.Mock()
scalyr_client._set_last_timestamp(0)
add_events_request = session.add_events_request()
ts = 2000
expected = str(ts + 1)
add_events_request.add_event(Event().set_message("eventOne"),
timestamp=ts)
add_events_request.add_event(Event().set_message("eventTwo"),
timestamp=1)
json = test_util.parse_scalyr_request(add_events_request.get_payload())
event = json["events"][1]
self.assertEquals(event["ts"], expected)
def test_only_message(self):
x = Event()
x.set_message("my_message")
output_buffer = StringIO()
x.serialize(output_buffer)
self.assertEquals("{attrs:{message:`s\x00\x00\x00\nmy_message}}", output_buffer.getvalue())
def test_multiple_calls_to_get_payload(self):
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertTrue(request.add_event(Event().set_message("eventOne"), timestamp=1))
self.assertTrue(request.add_event(Event().set_message("eventTwo"), timestamp=2))
self.assertEquals(request.get_payload(), request.get_payload())
request.close()
def test_maximum_bytes_exceeded(self):
request = AddEventsRequest(self.__body, max_size=112)
request.set_client_time(1)
self.assertTrue(request.add_event(Event().set_message('eventOne'), timestamp=1L))
self.assertFalse(request.add_event(Event().set_message('eventTwo'), timestamp=2L))
self.assertEquals(
request.get_payload(),
"""{"token":"fakeToken", events: [{attrs:{message:`s\x00\x00\x00\x08eventOne},ts:"1"}], logs: [], threads: [], client_time: 1 }""")
request.close()
def test_no_monotonically_increasing_timestamp(self):
request = AddEventsRequest(self.__body)
ts = 2000
self.assertTrue(
request.add_event(Event().set_message("eventOne"), timestamp=ts))
self.assertTrue(
request.add_event(Event().set_message("eventTwo"), timestamp=1))
json = test_util.parse_scalyr_request(request.get_payload())
event = json["events"][1]
self.assertEquals(event["ts"], "1")
def test_fast_path_fields(self):
x = Event(thread_id="foo", attrs={"parser": "bar"})
x.set_message(b"my_message")
x.set_sequence_number_delta(3)
x.set_timestamp(42)
output_buffer = BytesIO()
x.serialize(output_buffer)
self.assertEquals(
b'{thread:"foo", log:"foo", attrs:{"parser":"bar",message:`s\x00\x00\x00\nmy_message},sd:3,ts:"42"}',
output_buffer.getvalue(),
)
def test_same_sequence_id( self ):
expected_id = '1234'
expected_number = 1234
expected_delta = 1
event = Event()
self.event_sequencer.add_sequence_fields( event, expected_id, expected_number )
event = Event()
self.event_sequencer.add_sequence_fields( event, expected_id, expected_number + expected_delta )
self.assertIsNone( event.sequence_id )
self.assertIsNone( event.sequence_number)
self.assertEqual( expected_delta, event.sequence_number_delta )
def test_basic_case(self):
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertEquals(request.total_events, 0)
self.assertTrue(request.add_event(Event().set_message("eventOne"), timestamp=1L))
self.assertTrue(request.add_event(Event().set_message("eventTwo"), timestamp=2L))
self.assertEquals(
request.get_payload(),
"""{"token":"fakeToken", events: [{attrs:{message:`s\x00\x00\x00\x08eventOne},ts:"1"},{attrs:{message:`s\x00\x00\x00\x08eventTwo},ts:"2"}]"""
""", logs: [], threads: [], client_time: 1 }""")
self.assertEquals(request.total_events, 2)
request.close()
def test_set_position(self):
request = AddEventsRequest(self.__body)
request.set_client_time(1)
position = request.position()
self.assertTrue(request.add_event(Event().set_message('eventOne'), timestamp=1L))
self.assertTrue(request.add_event(Event().set_message('eventTwo'), timestamp=2L))
request.set_position(position)
self.assertTrue(request.add_event(Event().set_message('eventThree'), timestamp=3L))
self.assertEquals(
request.get_payload(),
"""{"token":"fakeToken", events: [{attrs:{message:`s\x00\x00\x00\neventThree},ts:"3"}], logs: [], threads: [], client_time: 1 }""")
request.close()
def test_fast_path_fields(self):
x = Event(thread_id='foo', attrs={"parser": "bar"})
x.set_message("my_message")
x.set_sequence_number_delta(3)
x.set_timestamp(42L)
output_buffer = StringIO()
x.serialize(output_buffer)
self.assertEquals(
'{thread:"foo", attrs:{"parser":"bar",message:`s\x00\x00\x00\nmy_message},sd:3,ts:"42"}',
output_buffer.getvalue())
def test_same_sequence_id( self ):
expected_id = '1234'
expected_number = 1234
expected_delta = 1
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertTrue(request.add_event(Event().set_message('eventOne'), timestamp=1L, sequence_id=expected_id, sequence_number=expected_number))
def test_send_request_body_is_logged_raw_uncompressed_long_body_is_truncated(
self):
# Verify that very large bodies are truncated to avoid increased memory usage issues under
# Python 2.7
session = ScalyrClientSession("https://dummserver.com",
"DUMMY API KEY", SCALYR_VERSION)
session._ScalyrClientSession__connection = mock.Mock()
session._ScalyrClientSession__receive_response = mock.Mock()
session._ScalyrClientSession__compress = mock.Mock(
return_value="compressed")
add_events_request = AddEventsRequest({"bar": "baz"})
event1 = Event(thread_id="foo4", attrs={
"parser": "bar2"
}).set_message("a" * (MAX_REQUEST_BODY_SIZE_LOG_MSG_LIMIT + 1))
add_events_request.add_event(event=event1, timestamp=1)
session.send(add_events_request=add_events_request)
# Should log raw (uncompressed) request body / payload
expected_body = (
r'Sending POST /addEvents with body "{"bar":"baz".*\.\.\. \[body truncated to %s chars\] \.\.\.'
% (MAX_REQUEST_BODY_SIZE_LOG_MSG_LIMIT))
self.assertLogFileContainsRegex(expected_body,
file_path=self.agent_debug_log_path)
def test_sequence_number_but_no_id(self):
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertTrue(
request.add_event(Event().set_message("eventOne"),
timestamp=1L,
sequence_number=1234))
def test_sequence_number_but_no_id(self):
event = Event()
self.event_sequencer.add_sequence_fields(event, None, 1234)
self.assertIsNone(event.sequence_id)
self.assertIsNone(event.sequence_number)
self.assertIsNone(event.sequence_number_delta)
def test_different_sequence_id( self ):
first_id = '1234'
second_id = '1235'
first_number = 1234
second_number = 1234
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertTrue(request.add_event(Event().set_message('eventOne'), timestamp=1L, sequence_id=first_id, sequence_number=first_number))
def test_monotonically_increasing_timestamp(self):
request = AddEventsRequest(self.__body,
enforce_monotonic_timestamps=True)
scalyr_client._set_last_timestamp(0)
ts = 2000
expected = str(ts + 1)
self.assertTrue(
request.add_event(Event().set_message("eventOne"), timestamp=ts))
self.assertTrue(
request.add_event(Event().set_message("eventTwo"), timestamp=1))
json = test_util.parse_scalyr_request(request.get_payload())
event = json["events"][1]
self.assertEquals(event["ts"], expected)
def test_set_client_time(self):
request = AddEventsRequest(self.__body)
request.set_client_time(100)
self.assertTrue(request.add_event(Event().set_message('eventOne'), timestamp=1L))
self.assertTrue(request.add_event(Event().set_message('eventTwo'), timestamp=2L))
self.assertEquals(
request.get_payload(),
"""{"token":"fakeToken", events: [{attrs:{message:`s\x00\x00\x00\x08eventOne},ts:"1"},{attrs:{message:`s\x00\x00\x00\x08eventTwo},ts:"2"}]"""
""", logs: [], threads: [], client_time: 100 }""")
request.set_client_time(2)
self.assertEquals(
request.get_payload(),
"""{"token":"fakeToken", events: [{attrs:{message:`s\x00\x00\x00\x08eventOne},ts:"1"},{attrs:{message:`s\x00\x00\x00\x08eventTwo},ts:"2"}]"""
""", logs: [], threads: [], client_time: 2 }""")
request.close()
def test_set_position_resets_sequence_compression( self ):
first_id = '1234'
second_id = '1235'
first_number = 1234
second_number = 4321
expected_delta = 10
request = AddEventsRequest(self.__body )
request.set_client_time(1)
self.assertTrue(request.add_event(Event().set_message('eventOne'), timestamp=1L, sequence_id=first_id, sequence_number=first_number))
def test_exceeds_size_doesnt_effect_sequence( self ):
first_id = '1234'
second_id = '1235'
first_number = 1234
second_number = 4321
expected_delta = 10
request = AddEventsRequest(self.__body, max_size=180)
request.set_client_time(1)
self.assertTrue(request.add_event(Event().set_message('eventOne'), timestamp=1L, sequence_id=first_id, sequence_number=first_number))
def test_add_log_and_thread(self):
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertEquals(request.total_events, 0)
self.assertTrue(request.add_event(Event().set_message("eventOne"), timestamp=1L))
self.assertTrue(request.add_log_and_thread('t1', 'n1', {"l1": "L1"}))
self.assertTrue(request.add_event(Event().set_message("eventTwo"), timestamp=2L))
self.assertTrue(request.add_log_and_thread('t2', 'n2', {"l2": "L2"}))
self.assertEquals(
request.get_payload(),
"""{"token":"fakeToken", events: [{attrs:{message:`s\x00\x00\x00\x08eventOne},ts:"1"},{attrs:{message:`s\x00\x00\x00\x08eventTwo},ts:"2"}]"""
""", logs: [{"attrs":{"l1":"L1"},"id":"t1"},{"attrs":{"l2":"L2"},"id":"t2"}], threads: [{"id":"t1","name":"n1"},{"id":"t2","name":"n2"}], client_time: 1 }""")
self.assertEquals(request.total_events, 2)
request.close()
def test_reset(self):
expected_id = "1234"
expected_number = 1234
expected_delta = 1
event = Event()
self.event_sequencer.add_sequence_fields(event, expected_id,
expected_number)
self.event_sequencer.reset()
event = Event()
self.event_sequencer.add_sequence_fields(
event, expected_id, expected_number + expected_delta)
self.assertEqual(expected_id.encode("utf-8"), event.sequence_id)
self.assertEqual(expected_number + expected_delta,
event.sequence_number)
self.assertIsNone(event.sequence_number_delta)
def test_set_position_with_log_and_thread(self):
request = AddEventsRequest(self.__body)
request.set_client_time(1)
position = request.position()
request.add_log_and_thread('log1', 'Hi there', {})
self.assertTrue(request.add_event(Event().set_message('eventOne'), timestamp=1L))
self.assertTrue(request.add_event(Event().set_message('eventTwo'), timestamp=2L))
request.set_position(position)
self.assertTrue(request.add_log_and_thread('log2', 'Log two', {}))
self.assertTrue(request.add_event(Event().set_message('eventThree'), timestamp=3L))
self.assertEquals(
request.get_payload(),
"""{"token":"fakeToken", events: [{attrs:{message:`s\x00\x00\x00\neventThree},ts:"3"}], """
"""logs: [{"attrs":{},"id":"log2"}], threads: [{"id":"log2","name":"Log two"}], client_time: 1 }""")
request.close()
def test_different_sequence_id(self):
first_id = "1234"
second_id = "1235"
first_number = 1234
second_number = 1234
event = Event()
self.event_sequencer.add_sequence_fields(event, first_id, first_number)
event = Event()
self.event_sequencer.add_sequence_fields(event, first_id, first_number + 1)
event = Event()
self.event_sequencer.add_sequence_fields(event, second_id, second_number)
self.assertEquals(second_id, event.sequence_id)
self.assertEquals(second_number, event.sequence_number)
self.assertIsNone(event.sequence_number_delta)
def test_sequence_id_and_number(self):
expected_id = "1234"
expected_number = 1234
event = Event()
self.event_sequencer.add_sequence_fields(event, expected_id, expected_number)
self.assertEquals(expected_id, event.sequence_id)
self.assertEquals(expected_number, event.sequence_number)
self.assertIsNone(event.sequence_number_delta)
def test_exceeds_size_doesnt_effect_sequence(self):
first_id = "1234"
second_id = "1235"
first_number = 1234
second_number = 4321
expected_delta = 10
request = AddEventsRequest(self.__body, max_size=180)
request.set_client_time(1)
self.assertTrue(
request.add_event(
Event().set_message("eventOne"),
timestamp=1,
sequence_id=first_id,
sequence_number=first_number,
)
)
self.assertFalse(
request.add_event(
Event(
attrs={"name": "eventTwo", "long": "some really long text"}
).set_message("eventTwo"),
timestamp=2,
sequence_id=second_id,
sequence_number=second_number,
)
)
self.assertTrue(
request.add_event(
Event().set_message("eventThree"),
timestamp=3,
sequence_id=first_id,
sequence_number=first_number + expected_delta,
)
)
self.assertEquals(request.total_events, 2)
json = test_util.parse_scalyr_request(request.get_payload())
event = json["events"][1]
self.assertFalse("si" in event)
self.assertEquals(expected_delta, event["sd"])
self.assertFalse("sn" in event)
request.close()
def test_set_position_resets_sequence_compression(self):
first_id = "1234"
second_id = "1235"
first_number = 1234
second_number = 4321
expected_delta = 10
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertTrue(
request.add_event(
Event().set_message("eventOne"),
timestamp=1,
sequence_id=first_id,
sequence_number=first_number,
)
)
position = request.position()
self.assertTrue(
request.add_event(
Event().set_message("eventTwo"),
timestamp=2,
sequence_id=first_id,
sequence_number=first_number + expected_delta,
)
)
request.set_position(position)
self.assertTrue(
request.add_event(
Event().set_message("eventThree"),
timestamp=3,
sequence_id=first_id,
sequence_number=second_number,
)
)
self.assertEquals(request.total_events, 2)
json = test_util.parse_scalyr_request(request.get_payload())
event = json["events"][1]
self.assertEquals(second_number, event["sn"])
self.assertFalse("sd" in event)
request.close()
def test_sequence_id_and_number(self):
expected_id = "1234"
expected_number = 1234
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertTrue(
request.add_event(
Event().set_message("eventOne"),
timestamp=1L,
sequence_id=expected_id,
sequence_number=expected_number,
))
def test_different_sequence_id(self):
first_id = "1234"
second_id = "1235"
first_number = 1234
second_number = 1234
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertTrue(
request.add_event(
Event().set_message("eventOne"),
timestamp=1,
sequence_id=first_id,
sequence_number=first_number,
)
)
self.assertTrue(
request.add_event(
Event().set_message("eventTwo"),
timestamp=2,
sequence_id=first_id,
sequence_number=first_number + 1,
)
)
self.assertTrue(
request.add_event(
Event().set_message("eventThree"),
timestamp=3,
sequence_id=second_id,
sequence_number=second_number,
)
)
self.assertEquals(request.total_events, 3)
json = test_util.parse_scalyr_request(request.get_payload())
event = json["events"][2]
self.assertEquals(second_id, event["si"])
self.assertEquals(second_number, event["sn"])
self.assertFalse("sd" in event)
request.close()
def generate_add_events_request(num_events,
line_length,
attributes_count,
base_body=None):
# type: (int, int, int, Optional[dict]) -> AddEventsRequest
"""
Generate AddEventsRequest object with the num_events number of events with the same payload
payload (line length) size.
"""
if not base_body:
base_body = {
"token": "api key",
"session": "sessionbar",
"threads": []
}
add_events_request = AddEventsRequest(base_body=base_body)
for _ in range(0, num_events):
line = generate_random_line(length=line_length)
attributes = generate_random_dict(keys_count=attributes_count)
event = Event(thread_id=100)
event.set_message(line)
event.add_attributes(attributes)
add_events_request.add_event(event)
return add_events_request
def test_all_fields(self):
x = Event(thread_id="foo", attrs={"parser": "bar"})
x.set_message("my_message")
x.set_sampling_rate(0.5)
x.set_sequence_id(1)
x.set_sequence_number(2)
x.set_sequence_number_delta(3)
x.set_timestamp(42L)
output_buffer = StringIO()
x.serialize(output_buffer)
self.assertEquals(
'{thread:"foo", attrs:{"parser":"bar",message:`s\x00\x00\x00\nmy_message,sample_rate:0.5},ts:"42",si:"1",sn:2,sd:3}',
output_buffer.getvalue(),
)
self.assertEquals(
json_lib.JsonObject(
thread="foo",
ts="42",
si="1",
sn=2,
sd=3,
attrs=json_lib.JsonObject(parser="bar", message="my_message", sample_rate=0.5),
),
json_lib.parse(output_buffer.getvalue()),
)
def test_no_attrs(self):
x = Event(thread_id="biz")
x.set_message("my_message")
x.set_sampling_rate(0.5)
x.set_sequence_id(1)
x.set_sequence_number(2)
x.set_sequence_number_delta(3)
x.set_timestamp(42L)
output_buffer = StringIO()
x.serialize(output_buffer)
self.assertEquals(
'{thread:"biz", attrs:{message:`s\x00\x00\x00\nmy_message,sample_rate:0.5},ts:"42",si:"1",sn:2,sd:3}',
output_buffer.getvalue(),
)
def test_individual_fields(self):
# snd
x = Event(thread_id="foo", attrs={"parser": "bar"})
x.set_message("my_message")
x.set_sequence_number_delta(3)
output_buffer = StringIO()
x.serialize(output_buffer)
self.assertEquals(
'{thread:"foo", attrs:{"parser":"bar",message:`s\x00\x00\x00\nmy_message},sd:3}', output_buffer.getvalue()
)
# timestamp
x = Event(thread_id="foo", attrs={"parser": "bar"})
x.set_message("my_message")
x.set_timestamp(42)
output_buffer = StringIO()
x.serialize(output_buffer)
self.assertEquals(
'{thread:"foo", attrs:{"parser":"bar",message:`s\x00\x00\x00\nmy_message},ts:"42"}',
output_buffer.getvalue(),
)
# sampling_rate
x = Event(thread_id="foo", attrs={"parser": "bar"})
x.set_message("my_message")
x.set_sampling_rate(0.5)
output_buffer = StringIO()
x.serialize(output_buffer)
self.assertEquals(
'{thread:"foo", attrs:{"parser":"bar",message:`s\x00\x00\x00\nmy_message,sample_rate:0.5}}',
output_buffer.getvalue(),
)
# sid
x = Event(thread_id="foo", attrs={"parser": "bar"})
x.set_message("my_message")
x.set_sequence_id("hi")
output_buffer = StringIO()
x.serialize(output_buffer)
self.assertEquals(
'{thread:"foo", attrs:{"parser":"bar",message:`s\x00\x00\x00\nmy_message},si:"hi"}',
output_buffer.getvalue(),
)
# seq num
x = Event(thread_id="foo", attrs={"parser": "bar"})
x.set_message("my_message")
x.set_sequence_number(5)
output_buffer = StringIO()
x.serialize(output_buffer)
self.assertEquals(
'{thread:"foo", attrs:{"parser":"bar",message:`s\x00\x00\x00\nmy_message},sn:5}', output_buffer.getvalue()
)
def test_create_from_template(self):
x = Event(thread_id="foo", attrs={"parser": "bar"})
x = Event(base=x)
x.set_message("my_message")
x.set_sampling_rate(0.5)
x.set_sequence_id(1)
x.set_sequence_number(2)
x.set_sequence_number_delta(3)
x.set_timestamp(42L)
output_buffer = StringIO()
x.serialize(output_buffer)
self.assertEquals(
'{thread:"foo", attrs:{"parser":"bar",message:`s\x00\x00\x00\nmy_message,sample_rate:0.5},ts:"42",si:"1",sn:2,sd:3}',
output_buffer.getvalue(),
)