def test_rate_limit_no_write_rate(self): """ Drop log messages with log size > max_write_burst but do not drop following small log messages """ self.__log_path = tempfile.mktemp('.log') scalyr_logging.set_log_destination(use_disk=True, logs_directory=os.path.dirname( self.__log_path), agent_log_file_path=self.__log_path, max_write_burst=250, log_write_rate=20000) self.__logger = scalyr_logging.getLogger('scalyr_agent.agent_main') string_300 = 'a' * 300 self.__logger.info('First message') self.assertTrue(self.__log_contains('First message')) self.__logger.info('Dropped message %s', string_300) self.assertFalse(self.__log_contains('Dropped message')) self.__logger.info('Second message') self.assertTrue(self.__log_contains('Second message')) self.assertTrue( self.__log_contains('Warning, skipped writing 1 log lines'))
def test_rate_limit_throttle_write_rate(self): """ Drop log messages with log size > max_write_burst and other following log messages if log write rate is low. """ self.__log_path = tempfile.mktemp(".log") scalyr_logging.set_log_destination( use_disk=True, logs_directory=os.path.dirname(self.__log_path), agent_log_file_path=self.__log_path, max_write_burst=250, log_write_rate=0, ) self.__logger = scalyr_logging.getLogger("scalyr_agent.agent_main") string_300 = "a" * 300 self.__logger.info("First message") self.assertTrue(self.__log_contains("First message")) self.__logger.info("Dropped message %s", string_300) self.assertFalse(self.__log_contains("Dropped message")) self.__logger.info("Second message") self.assertFalse(self.__log_contains("Second message"))
def setUp(self): super(SyslogMonitorConnectTest, self).setUp() self.monitor = None self.sockets = [] # capture log output scalyr_logging.set_log_destination(use_stdout=True) scalyr_logging.set_log_level(scalyr_logging.DEBUG_LEVEL_0) self.logger = logging.getLogger( "scalyr_agent.builtin_monitors.syslog_monitor.syslog") self.logger.setLevel(logging.INFO) self.stream = StringIO() self.handler = logging.StreamHandler(self.stream) self.logger.addHandler(self.handler) # Allow tcp_buffer_size to be set to less than 2k for option in MonitorInformation.__monitor_info__[ "scalyr_agent.builtin_monitors.syslog_monitor"].config_options: if option.option_name == "tcp_buffer_size": option.min_value = 0 # hide stdout self.old = sys.stdout # Replace sys.stdout with 'dummy' StringIO. # We must have one more variable which points to our 'dummy' stream because # Pytest can replace 'sys.stdout' with its own stream, # so we will not be able to access 'dummy' stream after that. self.dummy_stream = StringIO() sys.stdout = self.dummy_stream
def test_rate_limit(self): self.__log_path = tempfile.mktemp('.log') scalyr_logging.set_log_destination(use_disk=True, logs_directory=os.path.dirname( self.__log_path), agent_log_file_path=self.__log_path, max_write_burst=250, log_write_rate=0) self.__logger = scalyr_logging.getLogger('scalyr_agent.agent_main') string_300 = '' for i in range(300): string_300 = '%sa' % string_300 self.__logger.info('First message') self.assertTrue(self.__log_contains('First message')) self.__logger.info('Dropped message %s', string_300) self.assertFalse(self.__log_contains('Dropped message')) self.__logger.info('Second message') self.assertTrue(self.__log_contains('Second message')) self.assertTrue( self.__log_contains('Warning, skipped writing 1 log lines'))
def setUp(self): self.__log_path = tempfile.mktemp('.log') scalyr_logging.set_log_destination(use_disk=True, logs_directory=os.path.dirname( self.__log_path), agent_log_file_path=self.__log_path) self.__logger = scalyr_logging.getLogger('scalyr_agent.agent_main')
def run_standalone_monitor(monitor_module, monitor_python_path, monitor_config, monitor_sample_interval, monitor_debug_level, global_config_path): """Runs a single plugin monitor instance. @param monitor_module: The name of the python module implementing the monitor. @param monitor_python_path: The python path to search to find the module. @param monitor_config: The monitor configuration object. @param monitor_sample_interval: The default to use for the sample interval. @param monitor_debug_level: The debug level to use for logging. @param global_config_path: The path to the agent.json global configuration file to use, or None if none was supplied. """ scalyr_logging.set_log_destination(use_stdout=True) scalyr_logging.set_log_level(monitor_debug_level) log.log(scalyr_logging.DEBUG_LEVEL_1, 'Attempting to run module %s', monitor_module) try: parsed_config = json_lib.parse(monitor_config) log.log(scalyr_logging.DEBUG_LEVEL_1, 'Parsed configuration successfully') except json_lib.JsonParseException, e: print >> sys.stderr, 'Failed to parse the monitor configuration as valid JSON: %s', str( e) return 1
def setUp(self): super(SyslogMonitorConnectTest, self).setUp() self.monitor = None self.sockets = [] # capture log output scalyr_logging.set_log_destination(use_stdout=True) scalyr_logging.set_log_level(scalyr_logging.DEBUG_LEVEL_0) self.logger = logging.getLogger( "scalyr_agent.builtin_monitors.syslog_monitor.syslog" ) self.logger.setLevel(logging.INFO) self.stream = StringIO() self.handler = logging.StreamHandler(self.stream) self.logger.addHandler(self.handler) # hide stdout self.old = sys.stdout # Replace sys.stdout with 'dummy' StringIO. # We must have one more variable which points to our 'dummy' stream because # Pytest can replace 'sys.stdout' with its own stream, # so we will not be able to access 'dummy' stream after that. self.dummy_stream = StringIO() sys.stdout = self.dummy_stream
def test_rate_limit_no_write_rate(self): """ Drop log messages with log size > max_write_burst but do not drop following small log messages """ max_write_burst = 500 log_write_rate = 20000 scalyr_logging.set_log_destination( use_disk=True, logs_directory=os.path.dirname(self.__log_path), agent_log_file_path=self.__log_path, max_write_burst=max_write_burst, log_write_rate=log_write_rate, ) self.__logger = scalyr_logging.getLogger("scalyr_agent.agent_main") self.__logger.set_keep_last_record(True) # NOTE: Actual value which is being used for the rate limitting is the formatted value # and that value contains much more information than the string we generate here. # This means that 450 + common formatted string data will aways be > 500 and we need to # make sure that "max_write_burst" value we use is large enough so formatted "First # message" and "Second message" (with a warning) fit in that value, otherwise depending # on the timing and fill rate, the test may fail. string_450 = "a" * 450 self.__logger.info("First message") self.assertLogFileContainsLineRegex(expression="First message") first_message_record = self.__logger.last_record self.assertEqual(first_message_record.message, "First message") self.__logger.info("Dropped message %s", string_450) self.assertLogFileDoesntContainsLineRegex(expression="Dropped message") self.__logger.info("Second message") second_message_record = self.__logger.last_record self.assertEqual(second_message_record.message, "Second message") # Verify that formatted first mesage + second message length is not larger then 500 # (max_write_burst) which would indicate invalid test which may intermediatly fail # depending on the test timing self.assertEqual(1, second_message_record.rate_limited_dropped_records) if (first_message_record.formatted_size + second_message_record.formatted_size) >= max_write_burst: self.fail( "Length of the formatted first and second mesage string is longer than " "%s bytes (max_write_burst). Increase max_write_burst used or update the " "strings otherwise tests may occasionally fail." % (max_write_burst)) self.assertLogFileContainsLineRegex(expression="Second message") expression = "Warning, skipped writing 1 log lines" self.assertLogFileContainsLineRegex(expression=expression) self.assertTrue(second_message_record.rate_limited_result) self.assertTrue(second_message_record.rate_limited_set)
def setUp(self): super(ScalyrLoggingTest, self).setUp() self.__log_path = tempfile.mktemp(".log") scalyr_logging.set_log_destination( use_disk=True, logs_directory=os.path.dirname(self.__log_path), agent_log_file_path=self.__log_path, ) self.__logger = scalyr_logging.getLogger("scalyr_agent.agent_main")
def setUp(self): # We need to reset the log destinations here because it is only at this point is stdout replaced with # whatever object is capturing stdout for this test case. scalyr_logging.set_log_destination(use_stdout=True) self.__setup_invoked = True # Enable keys sort for json.dumps to make it easier to assert on the serialized output scalyr_util.SORT_KEYS = True # NOTE: orjson doesn't support sort_keys so we fallback to implementation which supports it scalyr_util.set_json_lib("json")
def _init_test_environment( self, use_pipelining=False, config_data=None, disable_flow_control=False, ): pipeline_threshold = 1.1 if use_pipelining: pipeline_threshold = 0.0 if config_data is None: config_data = {} if "workers" not in config_data: workers = [] for i in range(self.workers_count - 1): worker_config = { "id": "key_id_%s" % i, "api_key": "key_%s" % i, } workers.append(worker_config) config_data["workers"] = workers config_data["default_sessions_per_worker"] = self.worker_sessions_count config_data[ "use_multiprocess_workers"] = self.use_multiprocessing_workers config_data["disable_max_send_rate_enforcement_overrides"] = True config_data["pipeline_threshold"] = pipeline_threshold config_data["implicit_agent_log_collection"] = False self._env_builder = TestEnvironBuilder() self._env_builder.init_agent_dirs() self._env_builder.init_config(config_data) scalyr_logging.set_log_destination( use_disk=True, logs_directory=six.text_type( self._env_builder.config.agent_log_path), agent_log_file_path="agent.log", agent_debug_log_file_suffix="_debug", ) scalyr_logging.__log_manager__.set_log_level( scalyr_logging.DEBUG_LEVEL_5) self._env_builder.config.disable_flow_control = disable_flow_control self._env_builder.config.skip_agent_log_change = False
def run_standalone_monitor(monitor_module, monitor_python_path, monitor_config, monitor_sample_interval): """Runs a single plugin monitor instance. @param monitor_module: The name of the python module implementing the monitor. @param monitor_python_path: The python path to search to find the module. @param monitor_config: The monitor configuration object. @param monitor_sample_interval: The default to use for the sample interval. """ scalyr_logging.set_log_destination(use_stdout=True) try: parsed_config = json_lib.parse(monitor_config) except json_lib.JsonParseException, e: print >>sys.stderr, 'Failed to parse the monitor configuration as valid JSON: %s', str(e) return 1
def setUp(self): self.monitor = None self.sockets = [] # capture log output scalyr_logging.set_log_destination(use_stdout=True) scalyr_logging.set_log_level(scalyr_logging.DEBUG_LEVEL_0) self.logger = logging.getLogger("scalyr_agent.builtin_monitors.syslog_monitor.syslog") self.logger.setLevel(logging.INFO) self.stream = StringIO() self.handler = logging.StreamHandler(self.stream) self.logger.addHandler(self.handler) # hide stdout self.old = sys.stdout sys.stdout = StringIO()
def setUp(self): super(BaseScalyrLogCaptureTestCase, self).setUp() self.logs_directory = tempfile.mkdtemp(suffix="agent-tests-log") scalyr_logging.set_log_destination( use_disk=True, logs_directory=self.logs_directory, agent_log_file_path="agent.log", agent_debug_log_file_suffix="_debug", ) scalyr_logging.__log_manager__.set_log_level( scalyr_logging.DEBUG_LEVEL_5) self.agent_log_path = os.path.join(self.logs_directory, "agent.log") self.agent_debug_log_path = os.path.join(self.logs_directory, "agent_debug.log")
def test_rate_limit(self): self.__log_path = tempfile.mktemp('.log') scalyr_logging.set_log_destination(use_disk=True, logs_directory=os.path.dirname(self.__log_path), agent_log_file_path=self.__log_path, max_write_burst=250, log_write_rate=0) self.__logger = scalyr_logging.getLogger('scalyr_agent.agent_main') string_300 = '' for i in range(300): string_300 = '%sa' % string_300 self.__logger.info('First message') self.assertTrue(self.__log_contains('First message')) self.__logger.info('Dropped message %s', string_300) self.assertFalse(self.__log_contains('Dropped message')) self.__logger.info('Second message') self.assertTrue(self.__log_contains('Second message')) self.assertTrue(self.__log_contains('Warning, skipped writing 1 log lines'))
def run_standalone_monitor(monitor_module, monitor_python_path, monitor_config, monitor_sample_interval, monitor_debug_level, global_config_path): """Runs a single plugin monitor instance. @param monitor_module: The name of the python module implementing the monitor. @param monitor_python_path: The python path to search to find the module. @param monitor_config: The monitor configuration object. @param monitor_sample_interval: The default to use for the sample interval. @param monitor_debug_level: The debug level to use for logging. @param global_config_path: The path to the agent.json global configuration file to use, or None if none was supplied. """ scalyr_logging.set_log_destination(use_stdout=True) scalyr_logging.set_log_level(monitor_debug_level) log.log(scalyr_logging.DEBUG_LEVEL_1, 'Attempting to run module %s', monitor_module) try: parsed_config = json_lib.parse(monitor_config) log.log(scalyr_logging.DEBUG_LEVEL_1, 'Parsed configuration successfully') except json_lib.JsonParseException, e: print >>sys.stderr, 'Failed to parse the monitor configuration as valid JSON: %s', str(e) return 1
# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------------------ # # Runs all the tests for the scalyr-agent-2 package. # # author: Steven Czerwinski <*****@*****.**> __author__ = '*****@*****.**' from scalyr_agent.__scalyr__ import scalyr_init import scalyr_agent.scalyr_logging as scalyr_logging scalyr_init() scalyr_logging.set_log_destination(use_stdout=True) from scalyr_agent.all_tests import run_all_tests if __name__ == '__main__': run_all_tests()
import traceback from distutils import spawn from optparse import OptionParser # TODO: The following two imports have been modified to facilitate Windows platforms if 'win32' != sys.platform: from pwd import getpwnam import urllib from __scalyr__ import scalyr_init, get_install_root, TARBALL_INSTALL, MSI_INSTALL, SCALYR_VERSION scalyr_init() from scalyr_agent.scalyr_logging import set_log_destination set_log_destination(use_stdout=True) from scalyr_agent.scalyr_client import ScalyrClientSession from scalyr_agent.configuration import Configuration from scalyr_agent.platform_controller import PlatformController import scalyr_agent.json_lib as json_lib def set_api_key(config, config_file_path, new_api_key): """Replaces the current api key in the file at 'config_file_path' with the value of 'new_api_key'. @param config: The Configuration object created by reading config_file_path. @param config_file_path: The full path to the configuration file. This file will be overwritten. @param new_api_key: The new value for the api key to write into the file. """
def setUp(self): self.__log_path = tempfile.mktemp('.log') scalyr_logging.set_log_destination(use_disk=True, logs_directory=os.path.dirname(self.__log_path), agent_log_file_path=self.__log_path) self.__logger = scalyr_logging.getLogger('scalyr_agent.agent_main')
def run_standalone_monitor( monitor_module, monitor_python_path, monitor_config, monitor_sample_interval, monitor_debug_level, global_config_path, ): """Runs a single plugin monitor instance. @param monitor_module: The name of the python module implementing the monitor. @param monitor_python_path: The python path to search to find the module. @param monitor_config: The monitor configuration object. @param monitor_sample_interval: The default to use for the sample interval. @param monitor_debug_level: The debug level to use for logging. @param global_config_path: The path to the agent.json global configuration file to use, or None if none was supplied. """ scalyr_logging.set_log_destination(use_stdout=True) scalyr_logging.set_log_level(monitor_debug_level) log.log(scalyr_logging.DEBUG_LEVEL_1, "Attempting to run module %s", monitor_module) try: # Needs to be json_lib.parse because it is parsing configuration parsed_config = scalyr_util.json_scalyr_config_decode(monitor_config) log.log(scalyr_logging.DEBUG_LEVEL_1, "Parsed configuration successfully") except JsonParseException as e: print( "Failed to parse the monitor configuration as valid JSON: %s", six.text_type(e), file=sys.stderr, ) return 1 parsed_config["module"] = monitor_module if "id" not in parsed_config: parsed_config["id"] = "" # noinspection PyUnusedLocal def handle_shutdown_signal(signum, frame): print("Signal received, stopping monitor...", file=sys.stdout) monitor.stop() for sig in (signal.SIGTERM, signal.SIGINT): signal.signal(sig, handle_shutdown_signal) try: if global_config_path is not None: controller = PlatformController.new_platform() paths = controller.default_paths global_config = Configuration(global_config_path, paths, log) global_config.parse() else: global_config = None monitor = MonitorsManager.build_monitor( parsed_config, monitor_python_path, float(monitor_sample_interval), global_config, ) log.log(scalyr_logging.DEBUG_LEVEL_1, "Constructed monitor") monitor.open_metric_log() log.log(scalyr_logging.DEBUG_LEVEL_1, "Starting monitor") monitor.start() while monitor.isAlive(): time.sleep(0.1) except BadMonitorConfiguration as e: print("Invalid monitor configuration: %s" % six.text_type(e), file=sys.stderr) return 0
def setUp(self): # We need to reset the log destinations here because it is only at this point is stdout replaced with # whatever object is capturing stdout for this test case. scalyr_logging.set_log_destination(use_stdout=True) self.__setup_invoked = True