def create_configuration(extra_toplevel_config=None):
"""Creates a blank configuration file with default values. Optionally overwrites top-level key/values.
Sets api_key to 'fake' unless defined in extra_toplevel_config
@param extra_toplevel_config: Dict of top-level key/value objects to overwrite.
@return: The configuration object
@rtype: Configuration
"""
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
toplevel_config = {'api_key': 'fake'}
if extra_toplevel_config:
toplevel_config.update(extra_toplevel_config)
fp = open(config_file, 'w')
fp.write(json_lib.serialize(JsonObject(**toplevel_config)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths, None)
config.parse()
# we need to delete the config dir when done
atexit.register(shutil.rmtree, config_dir)
return config
def __create_test_instance(self, configuration_logs_entry,
monitors_log_configs):
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
logs_json_array = JsonArray()
for entry in configuration_logs_entry:
logs_json_array.add(JsonObject(content=entry))
fp = open(config_file, 'w')
fp.write(
json_lib.serialize(JsonObject(api_key='fake',
logs=logs_json_array)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths)
config.parse()
self.__monitor_fake_instances = []
for monitor_log_config in monitors_log_configs:
self.__monitor_fake_instances.append(
FakeMonitor(monitor_log_config))
# noinspection PyTypeChecker
return CopyingManager(config, self.__monitor_fake_instances)
def create_configuration(extra_toplevel_config=None):
"""Creates a blank configuration file with default values. Optionally overwrites top-level key/values.
Sets api_key to 'fake' unless defined in extra_toplevel_config
@param extra_toplevel_config: Dict of top-level key/value objects to overwrite.
@return: The configuration object
@rtype: Configuration
"""
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
toplevel_config = {'api_key': 'fake'}
if extra_toplevel_config:
toplevel_config.update(extra_toplevel_config)
fp = open(config_file, 'w')
fp.write(json_lib.serialize(JsonObject(**toplevel_config)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2', '/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths, None)
config.parse()
# we need to delete the config dir when done
atexit.register(shutil.rmtree, config_dir)
return config
def __create_test_instance(self, config_monitors, platform_monitors):
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
monitors_json_array = JsonArray()
for entry in config_monitors:
monitors_json_array.add(JsonObject(content=entry))
fp = open(config_file, 'w')
fp.write(
json_lib.serialize(
JsonObject(api_key='fake', monitors=monitors_json_array)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths)
config.parse()
# noinspection PyTypeChecker
return MonitorsManager(config, FakePlatform(platform_monitors))
def __create_test_instance(self, configuration_logs_entry, monitors_log_configs):
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, "agentConfig.json")
config_fragments_dir = os.path.join(config_dir, "configs.d")
os.makedirs(config_fragments_dir)
logs_json_array = JsonArray()
for entry in configuration_logs_entry:
logs_json_array.add(JsonObject(content=entry))
fp = open(config_file, "w")
fp.write(json_lib.serialize(JsonObject(api_key="fake", logs=logs_json_array)))
fp.close()
default_paths = DefaultPaths(
"/var/log/scalyr-agent-2", "/etc/scalyr-agent-2/agent.json", "/var/lib/scalyr-agent-2"
)
config = Configuration(config_file, default_paths)
config.parse()
self.__monitor_fake_instances = []
for monitor_log_config in monitors_log_configs:
self.__monitor_fake_instances.append(FakeMonitor(monitor_log_config))
# noinspection PyTypeChecker
return CopyingManager(config, self.__monitor_fake_instances)
def create_configuration(extra_toplevel_config=None):
"""Creates a blank configuration file with default values. Optionally overwrites top-level key/values.
Sets api_key to 'fake' unless defined in extra_toplevel_config
@param extra_toplevel_config: Dict of top-level key/value objects to overwrite.
@return: The configuration object
@rtype: Configuration
"""
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, "agentConfig.json")
config_fragments_dir = os.path.join(config_dir, "configs.d")
os.makedirs(config_fragments_dir)
toplevel_config = {"api_key": "fake"}
if extra_toplevel_config:
toplevel_config.update(extra_toplevel_config)
fp = open(config_file, "w")
fp.write(scalyr_util.json_encode(toplevel_config))
fp.close()
default_paths = DefaultPaths(
"/var/log/scalyr-agent-2",
"/etc/scalyr-agent-2/agent.json",
"/var/lib/scalyr-agent-2",
)
config = Configuration(config_file, default_paths, None)
config.parse()
# we need to delete the config dir when done
atexit.register(shutil.rmtree, config_dir)
return config
def config_object(self): # type: () -> Configuration
"""
Get config object from the config file.
"""
platform = PlatformController.new_platform()
platform._install_type = self._installation_type
default_types = platform.default_paths
config = Configuration(six.text_type(self._agent_config_path),
default_types, None)
config.parse()
return config
def setUp(self):
super(AgentMainStatusHandlerTestCase, self).setUp()
self.data_path = tempfile.mkdtemp(suffix="agent-data-path")
self.status_format_file = os.path.join(self.data_path, STATUS_FORMAT_FILE)
default_paths = mock.Mock()
default_paths.agent_data_path = self.data_path
default_paths.agent_log_path = "agent.log"
config_file = os.path.join(BASE_DIR, "fixtures/configs/agent1.json")
config = Configuration(config_file, default_paths, None)
config.parse()
self.agent = ScalyrAgent(PlatformController())
self.agent._ScalyrAgent__config = config
def create_copying_manager(self, config):
if 'api_key' not in config:
config['api_key'] = 'fake'
f = open(self._config_file, "w")
if f:
f.write(scalyr_util.json_encode(config))
f.close()
default_paths = DefaultPaths(self._log_dir, self._config_file,
self._data_dir)
configuration = Configuration(self._config_file, default_paths, None)
configuration.parse()
self._manager = TestableCopyingManager(configuration, [])
self._controller = self._manager.controller
def __create_test_instance(self, use_pipelining=False):
tmp_dir = tempfile.mkdtemp()
config_dir = os.path.join(tmp_dir, "config")
data_dir = os.path.join(tmp_dir, "data")
log_dir = os.path.join(tmp_dir, "log")
os.mkdir(data_dir)
os.mkdir(config_dir)
os.mkdir(log_dir)
self.__test_log_file = os.path.join(tmp_dir, "test.log")
fp = open(self.__test_log_file, "w")
fp.close()
config_file = os.path.join(config_dir, "agentConfig.json")
config_fragments_dir = os.path.join(config_dir, "configs.d")
os.makedirs(config_fragments_dir)
logs_json_array = JsonArray()
logs_json_array.add(JsonObject(path=self.__test_log_file))
pipeline_threshold = 1.1
if use_pipelining:
pipeline_threshold = 0.0
fp = open(config_file, "w")
fp.write(
json_lib.serialize(
JsonObject(
api_key="fake",
logs=logs_json_array,
pipeline_threshold=pipeline_threshold,
)))
fp.close()
default_paths = DefaultPaths(log_dir, config_file, data_dir)
config = Configuration(config_file, default_paths, None)
config.parse()
# noinspection PyTypeChecker
self._controller = TestableCopyingManager(config, []).controller
return self._controller
def create_copying_manager(self, config, monitor_agent_log=False):
if "api_key" not in config:
config["api_key"] = "fake"
if not monitor_agent_log:
config["implicit_agent_log_collection"] = False
f = open(self._config_file, "w")
if f:
f.write(scalyr_util.json_encode(config))
f.close()
default_paths = DefaultPaths(self._log_dir, self._config_file,
self._data_dir)
configuration = Configuration(self._config_file, default_paths, None)
configuration.parse()
self._manager = TestableCopyingManager(configuration, [])
self._controller = self._manager.controller
def __create_test_instance(self, config_monitors, platform_monitors):
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
monitors_json_array = JsonArray()
for entry in config_monitors:
monitors_json_array.add(JsonObject(content=entry))
fp = open(config_file, 'w')
fp.write(json_lib.serialize(JsonObject(api_key='fake', monitors=monitors_json_array)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2', '/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths)
config.parse()
# noinspection PyTypeChecker
return MonitorsManager(config, FakePlatform(platform_monitors))
def __create_test_instance(self, use_pipelining=False):
tmp_dir = tempfile.mkdtemp()
config_dir = os.path.join(tmp_dir, 'config')
data_dir = os.path.join(tmp_dir, 'data')
log_dir = os.path.join(tmp_dir, 'log')
os.mkdir(data_dir)
os.mkdir(config_dir)
os.mkdir(log_dir)
self.__test_log_file = os.path.join(tmp_dir, 'test.log')
fp = open(self.__test_log_file, 'w')
fp.close()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
logs_json_array = JsonArray()
logs_json_array.add(JsonObject(path=self.__test_log_file))
pipeline_threshold = 1.1
if use_pipelining:
pipeline_threshold = 0.0
fp = open(config_file, 'w')
fp.write(json_lib.serialize(JsonObject(api_key='fake', logs=logs_json_array,
pipeline_threshold=pipeline_threshold)))
fp.close()
default_paths = DefaultPaths(log_dir, config_file, data_dir)
config = Configuration(config_file, default_paths)
config.parse()
# noinspection PyTypeChecker
self._controller = TestableCopyingManager(config, []).controller
return self._controller
def test_config_parse_memory_leak(self):
# There was a memory leak with config.parse() and underlying __perform_substitutions method
config_path = os.path.join(BASE_DIR, "fixtures/configs/agent2.json")
default_paths = DefaultPaths(
"/var/log/scalyr-agent-2",
"/etc/scalyr-agent-2/agent.json",
"/var/lib/scalyr-agent-2",
)
# New config object is created
for index in range(0, 50):
config = Configuration(file_path=config_path,
default_paths=default_paths,
logger=None)
config.parse()
self.assertNoNewGarbage()
# Existing config object is reused
config = Configuration(file_path=config_path,
default_paths=default_paths,
logger=None)
for index in range(0, 50):
config.parse()
self.assertNoNewGarbage()
if 'id' not in parsed_config:
parsed_config['id'] = ''
# noinspection PyUnusedLocal
def handle_shutdown_signal(signum, frame):
print >>sys.stdout, 'Signal received, stopping monitor...'
monitor.stop()
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, handle_shutdown_signal)
try:
controller = PlatformController.new_platform()
paths = controller.default_paths
global_config = Configuration( paths.config_file_path, paths )
global_config.parse()
monitor = MonitorsManager.build_monitor(parsed_config, monitor_python_path, float(monitor_sample_interval), global_config )
log.log(scalyr_logging.DEBUG_LEVEL_1, 'Constructed monitor')
monitor.open_metric_log()
log.log(scalyr_logging.DEBUG_LEVEL_1, 'Starting monitor')
monitor.start()
while monitor.isAlive():
time.sleep(0.1)
except BadMonitorConfiguration, e:
print >>sys.stderr, 'Invalid monitor configuration: %s' % str(e)
return 0
if __name__ == '__main__':
parser = OptionParser(usage='Usage: python -m scalyr_agent.scalyr_monitor [options] monitor_module')
def run_standalone_monitor(
monitor_module,
monitor_python_path,
monitor_config,
monitor_sample_interval,
monitor_debug_level,
global_config_path,
):
"""Runs a single plugin monitor instance.
@param monitor_module: The name of the python module implementing the monitor.
@param monitor_python_path: The python path to search to find the module.
@param monitor_config: The monitor configuration object.
@param monitor_sample_interval: The default to use for the sample interval.
@param monitor_debug_level: The debug level to use for logging.
@param global_config_path: The path to the agent.json global configuration file to use, or None if none was
supplied.
"""
scalyr_logging.set_log_destination(use_stdout=True)
scalyr_logging.set_log_level(monitor_debug_level)
log.log(scalyr_logging.DEBUG_LEVEL_1, "Attempting to run module %s",
monitor_module)
try:
# Needs to be json_lib.parse because it is parsing configuration
parsed_config = scalyr_util.json_scalyr_config_decode(monitor_config)
log.log(scalyr_logging.DEBUG_LEVEL_1,
"Parsed configuration successfully")
except JsonParseException as e:
print(
"Failed to parse the monitor configuration as valid JSON: %s",
six.text_type(e),
file=sys.stderr,
)
return 1
parsed_config["module"] = monitor_module
if "id" not in parsed_config:
parsed_config["id"] = ""
# noinspection PyUnusedLocal
def handle_shutdown_signal(signum, frame):
print("Signal received, stopping monitor...", file=sys.stdout)
monitor.stop()
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, handle_shutdown_signal)
try:
if global_config_path is not None:
controller = PlatformController.new_platform()
paths = controller.default_paths
global_config = Configuration(global_config_path, paths, log)
global_config.parse()
else:
global_config = None
monitor = MonitorsManager.build_monitor(
parsed_config,
monitor_python_path,
float(monitor_sample_interval),
global_config,
)
log.log(scalyr_logging.DEBUG_LEVEL_1, "Constructed monitor")
monitor.open_metric_log()
log.log(scalyr_logging.DEBUG_LEVEL_1, "Starting monitor")
monitor.start()
while monitor.isAlive():
time.sleep(0.1)
except BadMonitorConfiguration as e:
print("Invalid monitor configuration: %s" % six.text_type(e),
file=sys.stderr)
return 0
parser.print_help(sys.stderr)
sys.exit(1)
if options.executing_user and getpass.getuser() != 'root':
print >> sys.stderr, 'You must be root to update the user account that is used to run the agent.'
sys.exit(1)
if options.config_filename is None:
options.config_filename = Configuration.default_config_file_path()
if not os.path.isabs(options.config_filename):
options.config_filename = os.path.abspath(options.config_filename)
try:
config_file = Configuration(options.config_filename, None, None)
config_file.parse()
except Exception, e:
print >> sys.stderr, 'Error reading configuration file: %s' % str(e)
print >> sys.stderr, traceback.format_exc()
print >> sys.stderr, 'Terminating, please fix the configuration file and restart agent.'
sys.exit(1)
if options.set_key_from_stdin:
api_key = raw_input('Please enter key: ')
set_api_key(config_file, options.config_filename, api_key)
elif options.api_key is not None:
set_api_key(config_file, options.config_filename, options.api_key)
if options.executing_user is not None:
set_executing_user(config_file, options.config_filename, options.executing_user)
parsed_config['id'] = ''
# noinspection PyUnusedLocal
def handle_shutdown_signal(signum, frame):
print >>sys.stdout, 'Signal received, stopping monitor...'
monitor.stop()
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, handle_shutdown_signal)
try:
if global_config_path is not None:
controller = PlatformController.new_platform()
paths = controller.default_paths
global_config = Configuration( global_config_path, paths )
global_config.parse(logger=log)
else:
global_config = None
monitor = MonitorsManager.build_monitor(parsed_config, monitor_python_path, float(monitor_sample_interval),
global_config )
log.log(scalyr_logging.DEBUG_LEVEL_1, 'Constructed monitor')
monitor.open_metric_log()
log.log(scalyr_logging.DEBUG_LEVEL_1, 'Starting monitor')
monitor.start()
while monitor.isAlive():
time.sleep(0.1)
except BadMonitorConfiguration, e:
print >>sys.stderr, 'Invalid monitor configuration: %s' % str(e)
return 0
'%s' % template)
sys.exit(1)
# Copy the file.
shutil.copy(template, config_path)
controller.set_file_owner(config_path,
controller.get_current_user())
print 'Successfully initialized the configuration file.'
if options.executing_user and controller.get_current_user() != 'root':
print >> sys.stderr, 'You must be root to update the user account that is used to run the agent.'
sys.exit(1)
try:
config_file = Configuration(options.config_filename, default_paths)
config_file.parse()
except Exception, e:
print >> sys.stderr, 'Error reading configuration file: %s' % str(e)
print >> sys.stderr, traceback.format_exc()
print >> sys.stderr, 'Terminating, please fix the configuration file and restart agent.'
sys.exit(1)
controller.consume_config(config_file, options.config_filename)
# See if we have to start the agent. This is only used by Windows right now as part of its install process.
if 'win32' == sys.platform and options.mark_conditional_restart:
mark_conditional_restart(controller, config_file)
if 'win32' == sys.platform and options.conditional_restart:
restart_if_conditional_marker_exists(controller, config_file)
