def _test_decompress_bytes(benchmark, compression_algorithm_tuple, log_tuple):
compression_algorithm, compression_level = compression_algorithm_tuple
file_name, bytes_to_read = log_tuple
data = read_bytes_from_log_fixture_file(file_name, bytes_to_read)
compress_func, _ = get_compress_and_decompress_func(
compression_algorithm, compression_level)
compressed_data = compress_func(data)
assert compressed_data != data
# NOTE: We intentionally request new decompression function so we get new zstandard context for
# decompression (this way we avoid dictionary being already populated).
_, decompress_func = get_compress_and_decompress_func(
compression_algorithm, compression_level)
def run_benchmark():
result = decompress_func(compressed_data)
return result
result = benchmark.pedantic(run_benchmark, iterations=10, rounds=20)
size_before_decompression = len(compressed_data)
size_after_decompression = len(result)
assert result is not None
# assert correctness
assert result != compressed_data
assert size_after_decompression > size_before_decompression
assert data == result
def main():
num_events = 40
line_length = 200
attributes_count = 3
add_events_request = generate_add_events_request(
num_events=num_events,
line_length=line_length,
attributes_count=attributes_count,
base_body=BASE_BODY,
)
add_events_data = add_events_request.get_payload()
for compression_type in SUPPORTED_COMPRESSION_ALGORITHMS:
print(("\n" + "=" * 100 + "\n"))
print(compression_type)
headers = copy.copy(BASE_HEADERS)
headers["Content-Encoding"] = compression_type
compression_level = COMPRESSION_TYPE_TO_DEFAULT_LEVEL[compression_type]
compress_func, _ = get_compress_and_decompress_func(
compression_type, compression_level)
data = compress_func(add_events_data)
assert len(data) < len(add_events_data)
res = requests.post(SCALYR_URL, headers=headers, data=data)
print(("Status code: %s" % (res.status_code)))
print(("Response body: %s" % (res.text)))
def _test_compress_bytes(benchmark, compression_algorithm_tuple, log_tuple):
compression_algorithm, compression_level = compression_algorithm_tuple
file_name, bytes_to_read = log_tuple
data = read_bytes_from_log_fixture_file(file_name, bytes_to_read)
compress_func, decompress_func = get_compress_and_decompress_func(
compression_algorithm, compression_level)
def run_benchmark():
result = compress_func(data)
return result
result = benchmark.pedantic(run_benchmark, iterations=10, rounds=20)
size_before_compression = len(data)
size_after_compression = len(result)
compression_ratio = float(size_before_compression) / size_after_compression
benchmark.stats.stats.size_before_compression = size_before_compression
benchmark.stats.stats.size_after_compression = size_after_compression
benchmark.stats.stats.compression_ratio = compression_ratio
assert result is not None
# assert correctness
assert size_after_compression < size_before_compression
assert data == decompress_func(result)
def test_compression_none(self):
data = self._data
compress = verify_and_get_compress_func("none")
self.assertIsNotNone(compress)
self.assertEqual(data, compress(data))
compress_func, decompress_func = get_compress_and_decompress_func(
"none")
self.assertIsNotNone(compress_func)
self.assertIsNotNone(decompress_func)
self.assertEqual(data, compress_func(data))
self.assertEqual(data, decompress_func(data))
self.assertEqual(data, decompress_func(compress_func(data)))
def test_send_request_body_compression(self):
add_events_request = AddEventsRequest({"bar": "baz"})
event1 = Event(thread_id="foo4", attrs={
"parser": "bar2"
}).set_message("test message 1")
event2 = Event(thread_id="foo5", attrs={
"parser": "bar2"
}).set_message("test message 2")
add_events_request.add_event(event=event1, timestamp=1)
add_events_request.add_event(event=event2, timestamp=2)
serialized_data = add_events_request.get_payload()
if sys.version_info < (2, 7, 0):
# lz4 and zstandard Python package is not available for Python 2.6
compression_types = scalyr_util.COMPRESSION_TYPE_TO_DEFAULT_LEVEL.copy(
)
del compression_types["zstandard"]
del compression_types["lz4"]
else:
compression_types = scalyr_util.COMPRESSION_TYPE_TO_DEFAULT_LEVEL
for compression_type in compression_types:
session = ScalyrClientSession(
"https://dummserver.com",
"DUMMY API KEY",
SCALYR_VERSION,
compression_type=compression_type,
)
session._ScalyrClientSession__connection = mock.Mock()
session._ScalyrClientSession__receive_response = mock.Mock()
session.send(add_events_request=add_events_request)
(
path,
request,
) = session._ScalyrClientSession__connection.post.call_args_list[0]
_, decompress_func = scalyr_util.get_compress_and_decompress_func(
compression_type)
self.assertEqual(path[0], "/addEvents")
if compression_type == "none":
self.assertTrue("Content-Encoding" not in
session._ScalyrClientSession__standard_headers)
self.assertTrue(b"test message 1" in request["body"])
self.assertTrue(b"test message 2" in request["body"])
else:
self.assertEqual(
session.
_ScalyrClientSession__standard_headers["Content-Encoding"],
compression_type,
)
# Verify decompressed data matches the raw body
self.assertTrue(b"test message 1" not in request["body"])
self.assertTrue(b"test message 2" not in request["body"])
self.assertFalse(serialized_data == request["body"])
self.assertEqual(serialized_data,
decompress_func(request["body"]))
# Compression is disabled
session = ScalyrClientSession(
"https://dummserver.com",
"DUMMY API KEY",
SCALYR_VERSION,
compression_type=None,
)
session._ScalyrClientSession__connection = mock.Mock()
session._ScalyrClientSession__receive_response = mock.Mock()
session.send(add_events_request=add_events_request)
serialized_data = add_events_request.get_payload()
(
path,
request,
) = session._ScalyrClientSession__connection.post.call_args_list[0]
_, decompress_func = scalyr_util.get_compress_and_decompress_func(
compression_type)
self.assertEqual(path[0], "/addEvents")
self.assertTrue(b"test message 1" in request["body"])
self.assertTrue(b"test message 2" in request["body"])
self.assertEqual(serialized_data, request["body"])
self.assertTrue("Content-Encoding" not in
session._ScalyrClientSession__standard_headers)