def create_configuration(extra_toplevel_config=None): """Creates a blank configuration file with default values. Optionally overwrites top-level key/values. Sets api_key to 'fake' unless defined in extra_toplevel_config @param extra_toplevel_config: Dict of top-level key/value objects to overwrite. @return: The configuration object @rtype: Configuration """ config_dir = tempfile.mkdtemp() config_file = os.path.join(config_dir, 'agentConfig.json') config_fragments_dir = os.path.join(config_dir, 'configs.d') os.makedirs(config_fragments_dir) toplevel_config = {'api_key': 'fake'} if extra_toplevel_config: toplevel_config.update(extra_toplevel_config) fp = open(config_file, 'w') fp.write(json_lib.serialize(JsonObject(**toplevel_config))) fp.close() default_paths = DefaultPaths('/var/log/scalyr-agent-2', '/etc/scalyr-agent-2/agent.json', '/var/lib/scalyr-agent-2') config = Configuration(config_file, default_paths, None) config.parse() # we need to delete the config dir when done atexit.register(shutil.rmtree, config_dir) return config
def __create_test_instance(self, configuration_logs_entry, monitors_log_configs): config_dir = tempfile.mkdtemp() config_file = os.path.join(config_dir, 'agentConfig.json') config_fragments_dir = os.path.join(config_dir, 'configs.d') os.makedirs(config_fragments_dir) logs_json_array = JsonArray() for entry in configuration_logs_entry: logs_json_array.add(JsonObject(content=entry)) fp = open(config_file, 'w') fp.write( json_lib.serialize(JsonObject(api_key='fake', logs=logs_json_array))) fp.close() default_paths = DefaultPaths('/var/log/scalyr-agent-2', '/etc/scalyr-agent-2/agent.json', '/var/lib/scalyr-agent-2') config = Configuration(config_file, default_paths) config.parse() self.__monitor_fake_instances = [] for monitor_log_config in monitors_log_configs: self.__monitor_fake_instances.append( FakeMonitor(monitor_log_config)) # noinspection PyTypeChecker return CopyingManager(config, self.__monitor_fake_instances)
def __create_test_instance(self, config_monitors, platform_monitors): config_dir = tempfile.mkdtemp() config_file = os.path.join(config_dir, 'agentConfig.json') config_fragments_dir = os.path.join(config_dir, 'configs.d') os.makedirs(config_fragments_dir) monitors_json_array = JsonArray() for entry in config_monitors: monitors_json_array.add(JsonObject(content=entry)) fp = open(config_file, 'w') fp.write( json_lib.serialize( JsonObject(api_key='fake', monitors=monitors_json_array))) fp.close() default_paths = DefaultPaths('/var/log/scalyr-agent-2', '/etc/scalyr-agent-2/agent.json', '/var/lib/scalyr-agent-2') config = Configuration(config_file, default_paths) config.parse() # noinspection PyTypeChecker return MonitorsManager(config, FakePlatform(platform_monitors))
def __create_test_instance(self, configuration_logs_entry, monitors_log_configs): config_dir = tempfile.mkdtemp() config_file = os.path.join(config_dir, "agentConfig.json") config_fragments_dir = os.path.join(config_dir, "configs.d") os.makedirs(config_fragments_dir) logs_json_array = JsonArray() for entry in configuration_logs_entry: logs_json_array.add(JsonObject(content=entry)) fp = open(config_file, "w") fp.write(json_lib.serialize(JsonObject(api_key="fake", logs=logs_json_array))) fp.close() default_paths = DefaultPaths( "/var/log/scalyr-agent-2", "/etc/scalyr-agent-2/agent.json", "/var/lib/scalyr-agent-2" ) config = Configuration(config_file, default_paths) config.parse() self.__monitor_fake_instances = [] for monitor_log_config in monitors_log_configs: self.__monitor_fake_instances.append(FakeMonitor(monitor_log_config)) # noinspection PyTypeChecker return CopyingManager(config, self.__monitor_fake_instances)
def create_configuration(extra_toplevel_config=None): """Creates a blank configuration file with default values. Optionally overwrites top-level key/values. Sets api_key to 'fake' unless defined in extra_toplevel_config @param extra_toplevel_config: Dict of top-level key/value objects to overwrite. @return: The configuration object @rtype: Configuration """ config_dir = tempfile.mkdtemp() config_file = os.path.join(config_dir, "agentConfig.json") config_fragments_dir = os.path.join(config_dir, "configs.d") os.makedirs(config_fragments_dir) toplevel_config = {"api_key": "fake"} if extra_toplevel_config: toplevel_config.update(extra_toplevel_config) fp = open(config_file, "w") fp.write(scalyr_util.json_encode(toplevel_config)) fp.close() default_paths = DefaultPaths( "/var/log/scalyr-agent-2", "/etc/scalyr-agent-2/agent.json", "/var/lib/scalyr-agent-2", ) config = Configuration(config_file, default_paths, None) config.parse() # we need to delete the config dir when done atexit.register(shutil.rmtree, config_dir) return config
def config_object(self): # type: () -> Configuration """ Get config object from the config file. """ platform = PlatformController.new_platform() platform._install_type = self._installation_type default_types = platform.default_paths config = Configuration(six.text_type(self._agent_config_path), default_types, None) config.parse() return config
def setUp(self): super(AgentMainStatusHandlerTestCase, self).setUp() self.data_path = tempfile.mkdtemp(suffix="agent-data-path") self.status_format_file = os.path.join(self.data_path, STATUS_FORMAT_FILE) default_paths = mock.Mock() default_paths.agent_data_path = self.data_path default_paths.agent_log_path = "agent.log" config_file = os.path.join(BASE_DIR, "fixtures/configs/agent1.json") config = Configuration(config_file, default_paths, None) config.parse() self.agent = ScalyrAgent(PlatformController()) self.agent._ScalyrAgent__config = config
def create_copying_manager(self, config): if 'api_key' not in config: config['api_key'] = 'fake' f = open(self._config_file, "w") if f: f.write(scalyr_util.json_encode(config)) f.close() default_paths = DefaultPaths(self._log_dir, self._config_file, self._data_dir) configuration = Configuration(self._config_file, default_paths, None) configuration.parse() self._manager = TestableCopyingManager(configuration, []) self._controller = self._manager.controller
def __create_test_instance(self, use_pipelining=False): tmp_dir = tempfile.mkdtemp() config_dir = os.path.join(tmp_dir, "config") data_dir = os.path.join(tmp_dir, "data") log_dir = os.path.join(tmp_dir, "log") os.mkdir(data_dir) os.mkdir(config_dir) os.mkdir(log_dir) self.__test_log_file = os.path.join(tmp_dir, "test.log") fp = open(self.__test_log_file, "w") fp.close() config_file = os.path.join(config_dir, "agentConfig.json") config_fragments_dir = os.path.join(config_dir, "configs.d") os.makedirs(config_fragments_dir) logs_json_array = JsonArray() logs_json_array.add(JsonObject(path=self.__test_log_file)) pipeline_threshold = 1.1 if use_pipelining: pipeline_threshold = 0.0 fp = open(config_file, "w") fp.write( json_lib.serialize( JsonObject( api_key="fake", logs=logs_json_array, pipeline_threshold=pipeline_threshold, ))) fp.close() default_paths = DefaultPaths(log_dir, config_file, data_dir) config = Configuration(config_file, default_paths, None) config.parse() # noinspection PyTypeChecker self._controller = TestableCopyingManager(config, []).controller return self._controller
def create_copying_manager(self, config, monitor_agent_log=False): if "api_key" not in config: config["api_key"] = "fake" if not monitor_agent_log: config["implicit_agent_log_collection"] = False f = open(self._config_file, "w") if f: f.write(scalyr_util.json_encode(config)) f.close() default_paths = DefaultPaths(self._log_dir, self._config_file, self._data_dir) configuration = Configuration(self._config_file, default_paths, None) configuration.parse() self._manager = TestableCopyingManager(configuration, []) self._controller = self._manager.controller
def __create_test_instance(self, config_monitors, platform_monitors): config_dir = tempfile.mkdtemp() config_file = os.path.join(config_dir, 'agentConfig.json') config_fragments_dir = os.path.join(config_dir, 'configs.d') os.makedirs(config_fragments_dir) monitors_json_array = JsonArray() for entry in config_monitors: monitors_json_array.add(JsonObject(content=entry)) fp = open(config_file, 'w') fp.write(json_lib.serialize(JsonObject(api_key='fake', monitors=monitors_json_array))) fp.close() default_paths = DefaultPaths('/var/log/scalyr-agent-2', '/etc/scalyr-agent-2/agent.json', '/var/lib/scalyr-agent-2') config = Configuration(config_file, default_paths) config.parse() # noinspection PyTypeChecker return MonitorsManager(config, FakePlatform(platform_monitors))
def __create_test_instance(self, use_pipelining=False): tmp_dir = tempfile.mkdtemp() config_dir = os.path.join(tmp_dir, 'config') data_dir = os.path.join(tmp_dir, 'data') log_dir = os.path.join(tmp_dir, 'log') os.mkdir(data_dir) os.mkdir(config_dir) os.mkdir(log_dir) self.__test_log_file = os.path.join(tmp_dir, 'test.log') fp = open(self.__test_log_file, 'w') fp.close() config_file = os.path.join(config_dir, 'agentConfig.json') config_fragments_dir = os.path.join(config_dir, 'configs.d') os.makedirs(config_fragments_dir) logs_json_array = JsonArray() logs_json_array.add(JsonObject(path=self.__test_log_file)) pipeline_threshold = 1.1 if use_pipelining: pipeline_threshold = 0.0 fp = open(config_file, 'w') fp.write(json_lib.serialize(JsonObject(api_key='fake', logs=logs_json_array, pipeline_threshold=pipeline_threshold))) fp.close() default_paths = DefaultPaths(log_dir, config_file, data_dir) config = Configuration(config_file, default_paths) config.parse() # noinspection PyTypeChecker self._controller = TestableCopyingManager(config, []).controller return self._controller
def test_config_parse_memory_leak(self): # There was a memory leak with config.parse() and underlying __perform_substitutions method config_path = os.path.join(BASE_DIR, "fixtures/configs/agent2.json") default_paths = DefaultPaths( "/var/log/scalyr-agent-2", "/etc/scalyr-agent-2/agent.json", "/var/lib/scalyr-agent-2", ) # New config object is created for index in range(0, 50): config = Configuration(file_path=config_path, default_paths=default_paths, logger=None) config.parse() self.assertNoNewGarbage() # Existing config object is reused config = Configuration(file_path=config_path, default_paths=default_paths, logger=None) for index in range(0, 50): config.parse() self.assertNoNewGarbage()
if 'id' not in parsed_config: parsed_config['id'] = '' # noinspection PyUnusedLocal def handle_shutdown_signal(signum, frame): print >>sys.stdout, 'Signal received, stopping monitor...' monitor.stop() for sig in (signal.SIGTERM, signal.SIGINT): signal.signal(sig, handle_shutdown_signal) try: controller = PlatformController.new_platform() paths = controller.default_paths global_config = Configuration( paths.config_file_path, paths ) global_config.parse() monitor = MonitorsManager.build_monitor(parsed_config, monitor_python_path, float(monitor_sample_interval), global_config ) log.log(scalyr_logging.DEBUG_LEVEL_1, 'Constructed monitor') monitor.open_metric_log() log.log(scalyr_logging.DEBUG_LEVEL_1, 'Starting monitor') monitor.start() while monitor.isAlive(): time.sleep(0.1) except BadMonitorConfiguration, e: print >>sys.stderr, 'Invalid monitor configuration: %s' % str(e) return 0 if __name__ == '__main__': parser = OptionParser(usage='Usage: python -m scalyr_agent.scalyr_monitor [options] monitor_module')
def run_standalone_monitor( monitor_module, monitor_python_path, monitor_config, monitor_sample_interval, monitor_debug_level, global_config_path, ): """Runs a single plugin monitor instance. @param monitor_module: The name of the python module implementing the monitor. @param monitor_python_path: The python path to search to find the module. @param monitor_config: The monitor configuration object. @param monitor_sample_interval: The default to use for the sample interval. @param monitor_debug_level: The debug level to use for logging. @param global_config_path: The path to the agent.json global configuration file to use, or None if none was supplied. """ scalyr_logging.set_log_destination(use_stdout=True) scalyr_logging.set_log_level(monitor_debug_level) log.log(scalyr_logging.DEBUG_LEVEL_1, "Attempting to run module %s", monitor_module) try: # Needs to be json_lib.parse because it is parsing configuration parsed_config = scalyr_util.json_scalyr_config_decode(monitor_config) log.log(scalyr_logging.DEBUG_LEVEL_1, "Parsed configuration successfully") except JsonParseException as e: print( "Failed to parse the monitor configuration as valid JSON: %s", six.text_type(e), file=sys.stderr, ) return 1 parsed_config["module"] = monitor_module if "id" not in parsed_config: parsed_config["id"] = "" # noinspection PyUnusedLocal def handle_shutdown_signal(signum, frame): print("Signal received, stopping monitor...", file=sys.stdout) monitor.stop() for sig in (signal.SIGTERM, signal.SIGINT): signal.signal(sig, handle_shutdown_signal) try: if global_config_path is not None: controller = PlatformController.new_platform() paths = controller.default_paths global_config = Configuration(global_config_path, paths, log) global_config.parse() else: global_config = None monitor = MonitorsManager.build_monitor( parsed_config, monitor_python_path, float(monitor_sample_interval), global_config, ) log.log(scalyr_logging.DEBUG_LEVEL_1, "Constructed monitor") monitor.open_metric_log() log.log(scalyr_logging.DEBUG_LEVEL_1, "Starting monitor") monitor.start() while monitor.isAlive(): time.sleep(0.1) except BadMonitorConfiguration as e: print("Invalid monitor configuration: %s" % six.text_type(e), file=sys.stderr) return 0
parser.print_help(sys.stderr) sys.exit(1) if options.executing_user and getpass.getuser() != 'root': print >> sys.stderr, 'You must be root to update the user account that is used to run the agent.' sys.exit(1) if options.config_filename is None: options.config_filename = Configuration.default_config_file_path() if not os.path.isabs(options.config_filename): options.config_filename = os.path.abspath(options.config_filename) try: config_file = Configuration(options.config_filename, None, None) config_file.parse() except Exception, e: print >> sys.stderr, 'Error reading configuration file: %s' % str(e) print >> sys.stderr, traceback.format_exc() print >> sys.stderr, 'Terminating, please fix the configuration file and restart agent.' sys.exit(1) if options.set_key_from_stdin: api_key = raw_input('Please enter key: ') set_api_key(config_file, options.config_filename, api_key) elif options.api_key is not None: set_api_key(config_file, options.config_filename, options.api_key) if options.executing_user is not None: set_executing_user(config_file, options.config_filename, options.executing_user)
parsed_config['id'] = '' # noinspection PyUnusedLocal def handle_shutdown_signal(signum, frame): print >>sys.stdout, 'Signal received, stopping monitor...' monitor.stop() for sig in (signal.SIGTERM, signal.SIGINT): signal.signal(sig, handle_shutdown_signal) try: if global_config_path is not None: controller = PlatformController.new_platform() paths = controller.default_paths global_config = Configuration( global_config_path, paths ) global_config.parse(logger=log) else: global_config = None monitor = MonitorsManager.build_monitor(parsed_config, monitor_python_path, float(monitor_sample_interval), global_config ) log.log(scalyr_logging.DEBUG_LEVEL_1, 'Constructed monitor') monitor.open_metric_log() log.log(scalyr_logging.DEBUG_LEVEL_1, 'Starting monitor') monitor.start() while monitor.isAlive(): time.sleep(0.1) except BadMonitorConfiguration, e: print >>sys.stderr, 'Invalid monitor configuration: %s' % str(e) return 0
'%s' % template) sys.exit(1) # Copy the file. shutil.copy(template, config_path) controller.set_file_owner(config_path, controller.get_current_user()) print 'Successfully initialized the configuration file.' if options.executing_user and controller.get_current_user() != 'root': print >> sys.stderr, 'You must be root to update the user account that is used to run the agent.' sys.exit(1) try: config_file = Configuration(options.config_filename, default_paths) config_file.parse() except Exception, e: print >> sys.stderr, 'Error reading configuration file: %s' % str(e) print >> sys.stderr, traceback.format_exc() print >> sys.stderr, 'Terminating, please fix the configuration file and restart agent.' sys.exit(1) controller.consume_config(config_file, options.config_filename) # See if we have to start the agent. This is only used by Windows right now as part of its install process. if 'win32' == sys.platform and options.mark_conditional_restart: mark_conditional_restart(controller, config_file) if 'win32' == sys.platform and options.conditional_restart: restart_if_conditional_marker_exists(controller, config_file)