def test_send_request_timestamp_increases_monotonically(self):
session = ScalyrClientSession(
"https://dummserver.com",
"DUMMY API KEY",
SCALYR_VERSION,
enforce_monotonic_timestamps=True,
)
session._ScalyrClientSession__connection = mock.Mock()
session._ScalyrClientSession__receive_response = mock.Mock()
scalyr_client._set_last_timestamp(0)
add_events_request = session.add_events_request()
ts = 2000
expected = str(ts + 1)
add_events_request.add_event(Event().set_message("eventOne"),
timestamp=ts)
add_events_request.add_event(Event().set_message("eventTwo"),
timestamp=1)
json = test_util.parse_scalyr_request(add_events_request.get_payload())
event = json["events"][1]
self.assertEquals(event["ts"], expected)
def test_get_user_agent_includes_requests_version(self):
scalyr_agent.scalyr_client.ssl.OPENSSL_VERSION_INFO = (1, 0, 2, 13, 13)
# without requests
session = ScalyrClientSession("https://dummserver.com",
"DUMMY API KEY", SCALYR_VERSION)
user_agent = session._ScalyrClientSession__standard_headers[
"User-Agent"]
split = user_agent.split(";")
self.assertEqual(split[-3], "o-1.0.2-13")
self.assertTrue(split[1].startswith("python-"))
# with requests
session = ScalyrClientSession(
"https://dummserver.com",
"DUMMY API KEY",
SCALYR_VERSION,
use_requests_lib=True,
)
user_agent = session._ScalyrClientSession__standard_headers[
"User-Agent"]
split = user_agent.split(";")
self.assertEqual(split[-1], "requests-2.15.1")
self.assertEqual(split[-4], "o-1.0.2-13")
self.assertTrue(split[1].startswith("python-"))
def test_user_agent_callback(self):
session = ScalyrClientSession("https://dummserver.com", "DUMMY API KEY", SCALYR_VERSION)
def get_user_agent():
return session._ScalyrClientSession__standard_headers['User-Agent']
base_ua = get_user_agent()
frags = ['frag1', 'frag2', 'frag3']
session.augment_user_agent(frags)
self.assertEquals(get_user_agent(), base_ua + ';' + ';'.join(frags))
def test_user_agent_callback(self):
session = ScalyrClientSession("https://dummserver.com", "DUMMY API KEY", SCALYR_VERSION)
def get_user_agent():
return session._ScalyrClientSession__standard_headers['User-Agent']
base_ua = get_user_agent()
frags = ['frag1', 'frag2', 'frag3']
session.augment_user_agent(frags)
self.assertEquals(get_user_agent(), base_ua + ';' + ';'.join(frags))
def test_get_user_agent_worker_and_api_key_info(self):
session = ScalyrClientSession(
"https://dummserver.com",
"DUMMY API KEY",
SCALYR_VERSION,
sessions_api_keys_tuple=None,
)
user_agent = session._ScalyrClientSession__standard_headers[
"User-Agent"]
split = user_agent.split(";")
self.assertTrue("mw-0" in split)
session = ScalyrClientSession(
"https://dummserver.com",
"DUMMY API KEY",
SCALYR_VERSION,
sessions_api_keys_tuple=("threaded", 1, 1),
)
user_agent = session._ScalyrClientSession__standard_headers[
"User-Agent"]
split = user_agent.split(";")
self.assertTrue("mw-0" in split)
session = ScalyrClientSession(
"https://dummserver.com",
"DUMMY API KEY",
SCALYR_VERSION,
sessions_api_keys_tuple=("threaded", 3, 2),
)
user_agent = session._ScalyrClientSession__standard_headers[
"User-Agent"]
split = user_agent.split(";")
self.assertTrue("mw-1|3|2" in split)
session = ScalyrClientSession(
"https://dummserver.com",
"DUMMY API KEY",
SCALYR_VERSION,
sessions_api_keys_tuple=("multiprocess", 4, 3),
)
user_agent = session._ScalyrClientSession__standard_headers[
"User-Agent"]
split = user_agent.split(";")
self.assertTrue("mw-2|4|3" in split)
def test_get_user_agent_string_run_as_admin(self, mock_new_platform):
mock_platform = mock.Mock()
mock_platform.get_current_user.return_value = "nobody"
mock_new_platform.return_value = mock_platform
session = ScalyrClientSession("https://dummserver.com",
"DUMMY API KEY", SCALYR_VERSION)
user_agent = session._ScalyrClientSession__standard_headers[
"User-Agent"]
split = user_agent.split(";")
self.assertTrue("a-0" in split)
mock_platform = mock.Mock()
mock_platform.get_current_user.return_value = "User"
mock_new_platform.return_value = mock_platform
session = ScalyrClientSession("https://dummserver.com",
"DUMMY API KEY", SCALYR_VERSION)
user_agent = session._ScalyrClientSession__standard_headers[
"User-Agent"]
split = user_agent.split(";")
self.assertFalse("" in split)
mock_platform = mock.Mock()
mock_platform.get_current_user.return_value = "root"
mock_new_platform.return_value = mock_platform
session = ScalyrClientSession("https://dummserver.com",
"DUMMY API KEY", SCALYR_VERSION)
user_agent = session._ScalyrClientSession__standard_headers[
"User-Agent"]
split = user_agent.split(";")
self.assertTrue("a-1" in split)
mock_platform = mock.Mock()
mock_platform.get_current_user.return_value = "MyDomain\\Administrators"
mock_new_platform.return_value = mock_platform
session = ScalyrClientSession("https://dummserver.com",
"DUMMY API KEY", SCALYR_VERSION)
user_agent = session._ScalyrClientSession__standard_headers[
"User-Agent"]
split = user_agent.split(";")
self.assertTrue("a-1" in split)
def test_send_request_body_is_logged_raw_uncompressed_long_body_is_truncated(
self):
# Verify that very large bodies are truncated to avoid increased memory usage issues under
# Python 2.7
session = ScalyrClientSession("https://dummserver.com",
"DUMMY API KEY", SCALYR_VERSION)
session._ScalyrClientSession__connection = mock.Mock()
session._ScalyrClientSession__receive_response = mock.Mock()
session._ScalyrClientSession__compress = mock.Mock(
return_value="compressed")
add_events_request = AddEventsRequest({"bar": "baz"})
event1 = Event(thread_id="foo4", attrs={
"parser": "bar2"
}).set_message("a" * (MAX_REQUEST_BODY_SIZE_LOG_MSG_LIMIT + 1))
add_events_request.add_event(event=event1, timestamp=1)
session.send(add_events_request=add_events_request)
# Should log raw (uncompressed) request body / payload
expected_body = (
r'Sending POST /addEvents with body "{"bar":"baz".*\.\.\. \[body truncated to %s chars\] \.\.\.'
% (MAX_REQUEST_BODY_SIZE_LOG_MSG_LIMIT))
self.assertLogFileContainsRegex(expected_body,
file_path=self.agent_debug_log_path)
def test_send_request_body_is_logged_raw_uncompressed(self):
"""
When sending a request with compression available / enabled, raw (uncompressed) request
body (payload) should be logged under DEBUG log level.
"""
session = ScalyrClientSession("https://dummserver.com",
"DUMMY API KEY", SCALYR_VERSION)
session._ScalyrClientSession__connection = mock.Mock()
session._ScalyrClientSession__receive_response = mock.Mock()
session._ScalyrClientSession__compress = mock.Mock(
return_value="compressed")
add_events_request = AddEventsRequest({"foo": "bar"})
event1 = Event(thread_id="foo1", attrs={
"parser": "bar1"
}).set_message("eventOne")
event2 = Event(thread_id="foo2", attrs={
"parser": "bar2"
}).set_message("eventTwo")
add_events_request.add_event(event=event1, timestamp=1)
add_events_request.add_event(event=event2, timestamp=2)
session.send(add_events_request=add_events_request)
# Should log raw (uncompressed) request body / payload
expected_body = r'{"foo":"bar", events: \[{thread:"foo1", .*'
self.assertLogFileContainsRegex(expected_body,
file_path=self.agent_debug_log_path)
expected_body = r'.*,{thread:"foo2", log:"foo2", attrs:{"parser":"bar2",.*'
self.assertLogFileContainsRegex(expected_body,
file_path=self.agent_debug_log_path)
# Verify that the compression was indeed enabled since that's the scenario we are testing
call_kwargs = session._ScalyrClientSession__connection.post.call_args_list[
0][1]
self.assertEqual(call_kwargs["body"], "compressed")
def upgrade_windows_install(config,
release_track="stable",
preserve_msi=False,
use_ui=True):
"""Performs an upgrade for an existing Scalyr Agent 2 that was previously installed using a Windows MSI install
file.
This will contact the Scalyr servers to see what the most up-to-date version of the agent is and, if necessary,
download an MSI file.
@param config: The configuration for this agent.
@param release_track: The release track to use when checking which version is the latest.
@param preserve_msi: Whether or not to delete the MSI file once the upgrade is finished. Note, this
argument is essentially ignored for now and we always leave the file because we cannot delete it with
the current way we exec the msiexec process.
@param use_ui: Whether or not the msiexec upgrade command should be run with the UI.
@rtype config: Configuration
@rtype release_track: str
@rtype preserve_msi: bool
@rtype use_ui: bool
@return: The exit status code.
"""
# The URL path of the agent to upgrade to.
url_path = None
try:
platform_controller = PlatformController.new_platform()
my_default_paths = platform_controller.default_paths
# Ensure agent was installed via MSI
if MSI_INSTALL != platform_controller.install_type:
raise UpgradeFailure(
'The current agent was not installed via MSI, so you may not use the upgrade windows '
'command.')
# Ensure that the user has not changed the defaults for the config, data, and log directory.
if my_default_paths.config_file_path != config.file_path:
raise UpgradeFailure(
'The agent is not using the default configuration file so you may not use the '
'upgrade windows command.')
if my_default_paths.agent_data_path != config.agent_data_path:
raise UpgradeFailure(
'The agent is not using the default data directory so you may not use the upgrade '
'windows command.')
if my_default_paths.agent_log_path != config.agent_log_path:
raise UpgradeFailure(
'The agent is not using the default log directory so you may not use the upgrade '
'windows command.')
# Determine if a newer version is available
client = ScalyrClientSession(config.scalyr_server,
config.api_key,
SCALYR_VERSION,
quiet=True,
ca_file=config.ca_cert_path)
status, size, response = client.perform_agent_version_check(
release_track)
if status.lower() != 'success':
raise UpgradeFailure(
'Failed to contact the Scalyr servers to check for latest update. Error code '
'was "%s"' % status)
# TODO: We shouldn't have to reparse response on JSON, but for now that, that's what the client library
# does.
data_payload = json_lib.parse(response)['data']
if not data_payload['update_required']:
print 'The latest version is already installed.'
return 0
print 'Attempting to upgrade agent from version %s to version %s.' % (
SCALYR_VERSION, data_payload['current_version'])
url_path = data_payload['urls']['win32']
file_portion = url_path[url_path.rfind('/') + 1:]
download_location = os.path.join(tempfile.gettempdir(), file_portion)
try:
try:
print 'Downloading agent from %s.' % url_path
urllib.urlretrieve(url_path, download_location)
if not os.path.isfile(download_location):
raise UpgradeFailure(
'Failed to download installation package')
if use_ui:
print(
'Executing upgrade. Please follow the instructions in the subsequent dialog boxes to '
'complete the upgrade process.')
else:
print(
'Executing upgrade. It will finish in the background.'
)
# Because this file, config_main.py, is part of the currently installed Scalyr Agent package, we have
# to finish our use of it before the upgrade can proceed. So, we just fork off the msiexec process
# in detached mode and terminate this program. This means we cannot report any errors that happen
# here, but I don't see a way around this for now.
# noinspection PyUnresolvedReferences
from win32process import DETACHED_PROCESS
upgrade_command = [
'msiexec.exe', '/i', "{}".format(download_location)
]
if not use_ui:
upgrade_command.append('/qn')
subprocess.Popen(upgrade_command,
shell=False,
stdin=None,
stdout=None,
stderr=None,
close_fds=True,
creationflags=DETACHED_PROCESS)
return 0
except IOError, error:
raise UpgradeFailure(
'Could not download the installer, returned error %s' %
str(error))
finally:
# TODO: Actually delete the temporary file. We cannot right now since our execution finishes
# before the msiexec process runs, but maybe we can do something like have a small shell script
# that runs the upgrader and then deletes the file. Something to consider post-alpha release.
if preserve_msi:
print 'Downloaded installer file has been left at %s' % download_location
except UpgradeFailure, error:
print >> sys.stderr
print >> sys.stderr, 'The upgrade failed due to the following reason: %s' % error.message
if url_path is not None:
print >> sys.stderr, 'You may try downloading and running the installer file yourself.'
print >> sys.stderr, 'The installer can be downloaded from %s' % url_path
print >> sys.stderr, 'Please e-mail [email protected] for help resolving this issue.'
return 1
def upgrade_windows_install(config, release_track="stable", preserve_msi=False, use_ui=True):
"""Performs an upgrade for an existing Scalyr Agent 2 that was previously installed using a Windows MSI install
file.
This will contact the Scalyr servers to see what the most up-to-date version of the agent is and, if necessary,
download an MSI file.
@param config: The configuration for this agent.
@param release_track: The release track to use when checking which version is the latest.
@param preserve_msi: Whether or not to delete the MSI file once the upgrade is finished. Note, this
argument is essentially ignored for now and we always leave the file because we cannot delete it with
the current way we exec the msiexec process.
@param use_ui: Whether or not the msiexec upgrade command should be run with the UI.
@rtype config: Configuration
@rtype release_track: str
@rtype preserve_msi: bool
@rtype use_ui: bool
@return: The exit status code.
"""
# The URL path of the agent to upgrade to.
url_path = None
try:
platform_controller = PlatformController.new_platform()
my_default_paths = platform_controller.default_paths
# Ensure agent was installed via MSI
if MSI_INSTALL != platform_controller.install_type:
raise UpgradeFailure('The current agent was not installed via MSI, so you may not use the upgrade windows '
'command.')
# Ensure that the user has not changed the defaults for the config, data, and log directory.
if my_default_paths.config_file_path != config.file_path:
raise UpgradeFailure('The agent is not using the default configuration file so you may not use the '
'upgrade windows command.')
if my_default_paths.agent_data_path != config.agent_data_path:
raise UpgradeFailure('The agent is not using the default data directory so you may not use the upgrade '
'windows command.')
if my_default_paths.agent_log_path != config.agent_log_path:
raise UpgradeFailure('The agent is not using the default log directory so you may not use the upgrade '
'windows command.')
# Determine if a newer version is available
client = ScalyrClientSession(config.scalyr_server, config.api_key, SCALYR_VERSION, quiet=True,
ca_file=config.ca_cert_path)
status, size, response = client.perform_agent_version_check(release_track)
if status.lower() != 'success':
raise UpgradeFailure('Failed to contact the Scalyr servers to check for latest update. Error code '
'was "%s"' % status)
# TODO: We shouldn't have to reparse response on JSON, but for now that, that's what the client library
# does.
data_payload = json_lib.parse(response)['data']
if not data_payload['update_required']:
print 'The latest version is already installed.'
return 0
print 'Attempting to upgrade agent from version %s to version %s.' % (SCALYR_VERSION,
data_payload['current_version'])
url_path = data_payload['urls']['win32']
file_portion = url_path[url_path.rfind('/')+1:]
download_location = os.path.join(tempfile.gettempdir(), file_portion)
try:
try:
print 'Downloading agent from %s.' % url_path
urllib.urlretrieve(url_path, download_location)
if not os.path.isfile(download_location):
raise UpgradeFailure('Failed to download installation package')
if use_ui:
print ('Executing upgrade. Please follow the instructions in the subsequent dialog boxes to '
'complete the upgrade process.')
else:
print ('Executing upgrade. It will finish in the background.')
# Because this file, config_main.py, is part of the currently installed Scalyr Agent package, we have
# to finish our use of it before the upgrade can proceed. So, we just fork off the msiexec process
# in detached mode and terminate this program. This means we cannot report any errors that happen
# here, but I don't see a way around this for now.
# noinspection PyUnresolvedReferences
from win32process import DETACHED_PROCESS
upgrade_command = ['msiexec.exe', '/i', "{}".format(download_location)]
if not use_ui:
upgrade_command.append('/qn')
subprocess.Popen(upgrade_command,
shell=False, stdin=None, stdout=None, stderr=None, close_fds=True,
creationflags=DETACHED_PROCESS)
return 0
except IOError, error:
raise UpgradeFailure('Could not download the installer, returned error %s' % str(error))
finally:
# TODO: Actually delete the temporary file. We cannot right now since our execution finishes
# before the msiexec process runs, but maybe we can do something like have a small shell script
# that runs the upgrader and then deletes the file. Something to consider post-alpha release.
if preserve_msi:
print 'Downloaded installer file has been left at %s' % download_location
except UpgradeFailure, error:
print >>sys.stderr
print >>sys.stderr, 'The upgrade failed due to the following reason: %s' % error.message
if url_path is not None:
print >>sys.stderr, 'You may try downloading and running the installer file yourself.'
print >>sys.stderr, 'The installer can be downloaded from %s' % url_path
print >>sys.stderr, 'Please e-mail [email protected] for help resolving this issue.'
return 1
def test_send_request_body_compression(self):
add_events_request = AddEventsRequest({"bar": "baz"})
event1 = Event(thread_id="foo4", attrs={
"parser": "bar2"
}).set_message("test message 1")
event2 = Event(thread_id="foo5", attrs={
"parser": "bar2"
}).set_message("test message 2")
add_events_request.add_event(event=event1, timestamp=1)
add_events_request.add_event(event=event2, timestamp=2)
serialized_data = add_events_request.get_payload()
if sys.version_info < (2, 7, 0):
# lz4 and zstandard Python package is not available for Python 2.6
compression_types = scalyr_util.COMPRESSION_TYPE_TO_DEFAULT_LEVEL.copy(
)
del compression_types["zstandard"]
del compression_types["lz4"]
else:
compression_types = scalyr_util.COMPRESSION_TYPE_TO_DEFAULT_LEVEL
for compression_type in compression_types:
session = ScalyrClientSession(
"https://dummserver.com",
"DUMMY API KEY",
SCALYR_VERSION,
compression_type=compression_type,
)
session._ScalyrClientSession__connection = mock.Mock()
session._ScalyrClientSession__receive_response = mock.Mock()
session.send(add_events_request=add_events_request)
(
path,
request,
) = session._ScalyrClientSession__connection.post.call_args_list[0]
_, decompress_func = scalyr_util.get_compress_and_decompress_func(
compression_type)
self.assertEqual(path[0], "/addEvents")
if compression_type == "none":
self.assertTrue("Content-Encoding" not in
session._ScalyrClientSession__standard_headers)
self.assertTrue(b"test message 1" in request["body"])
self.assertTrue(b"test message 2" in request["body"])
else:
self.assertEqual(
session.
_ScalyrClientSession__standard_headers["Content-Encoding"],
compression_type,
)
# Verify decompressed data matches the raw body
self.assertTrue(b"test message 1" not in request["body"])
self.assertTrue(b"test message 2" not in request["body"])
self.assertFalse(serialized_data == request["body"])
self.assertEqual(serialized_data,
decompress_func(request["body"]))
# Compression is disabled
session = ScalyrClientSession(
"https://dummserver.com",
"DUMMY API KEY",
SCALYR_VERSION,
compression_type=None,
)
session._ScalyrClientSession__connection = mock.Mock()
session._ScalyrClientSession__receive_response = mock.Mock()
session.send(add_events_request=add_events_request)
serialized_data = add_events_request.get_payload()
(
path,
request,
) = session._ScalyrClientSession__connection.post.call_args_list[0]
_, decompress_func = scalyr_util.get_compress_and_decompress_func(
compression_type)
self.assertEqual(path[0], "/addEvents")
self.assertTrue(b"test message 1" in request["body"])
self.assertTrue(b"test message 2" in request["body"])
self.assertEqual(serialized_data, request["body"])
self.assertTrue("Content-Encoding" not in
session._ScalyrClientSession__standard_headers)
