def parse(self):
self.__read_time = time.time()
try:
try:
# First read the file. This makes sure it exists and can be parsed.
self.__config = scalyr_util.read_file_as_json(self.__file_path)
# What implicit entries do we need to add? metric monitor, agent.log, and then logs from all monitors.
except JsonReadFileException, e:
raise BadConfiguration(str(e), None, 'fileParseError')
# Import any requested variables from the shell and use them for substitutions.
self.__import_shell_variables()
self.__perform_substitutions()
self.__verify_main_config_and_apply_defaults(self.__config, self.__file_path)
self.__verify_logs_and_monitors_configs_and_apply_defaults(self.__config, self.__file_path)
# Now, look for any additional configuration in the config fragment directory.
for fp in self.__list_files(self.config_directory):
self.__additional_paths.append(fp)
content = scalyr_util.read_file_as_json(fp)
for k in content.keys():
if k not in ('logs', 'monitors', 'server_attributes'):
self.__last_error = BadConfiguration(
'Configuration fragment file "%s" contains an invalid key "%s". The config files in the '
'configuration directory can only contain "logs", "monitors", and "server_attributes" '
'entries.' % (fp, k), k, 'badFragmentKey')
raise self.__last_error
self.__verify_logs_and_monitors_configs_and_apply_defaults(content, fp)
self.__add_elements_from_array('logs', content, self.__config)
self.__add_elements_from_array('monitors', content, self.__config)
self.__merge_server_attributes(fp, content, self.__config)
# Add in 'serverHost' to server_attributes if it is not set. We must do this after merging any
# server attributes from the config fragments.
if 'serverHost' not in self.server_attributes:
self.__config['server_attributes']['serverHost'] = socket.gethostname()
# Add in implicit entry to collect the log generated by this agent.
agent_log = None
if self.implicit_agent_log_collection:
config = JsonObject(path='agent.log')
self.__verify_log_entry_and_set_defaults(config, description='implicit rule')
agent_log = config
# Add in any platform-specific monitors.
platform_monitors = []
for monitor in self.__default_monitors:
config = JsonObject(content=monitor)
self.__verify_monitor_entry_and_set_defaults(config, 'default monitors for platform', -1)
platform_monitors.append(config)
all_logs = list(self.__config.get_json_array('logs'))
if agent_log is not None:
all_logs.append(agent_log)
# We need to go back and fill in the monitor id if it is not set. We do this by keeping a count of
# how many monitors we have with the same module name (just considering the last element of the module
# path). We use the shortened form of the module name because that is used when emitting lines for
# this monitor in the logs -- see scalyr_logging.py.
monitors_by_module_name = {}
# Tracks which modules already had an id present in the module config.
had_id = {}
all_monitors = list(self.__config.get_json_array('monitors'))
for monitor in platform_monitors:
all_monitors.append(monitor)
for entry in all_monitors:
module_name = entry['module'].split('.')[-1]
if not module_name in monitors_by_module_name:
index = 1
else:
index = monitors_by_module_name[module_name] + 1
if 'id' not in entry:
entry['id'] = index
else:
had_id[module_name] = True
monitors_by_module_name[module_name] = index
# Just as a simplification, if there is only one monitor with a given name, we remove the monitor_id
# to clean up it's name in the logs.
for entry in all_monitors:
module_name = entry['module'].split('.')[-1]
if monitors_by_module_name[module_name] == 1 and not module_name in had_id:
entry['id'] = ''
# Now build up __logs to have an object created for each log entry, and __monitors to have an object
# created for each monitor entry.
for entry in all_logs:
# Automatically add in the parser to the attributes section. We make a copy of the object first
# just to be safe.
entry = JsonObject(content=entry)
if 'parser' in entry:
entry['attributes']['parser'] = entry['parser']
if self.__log_factory is not None:
self.__logs.append(self.__log_factory(entry))
if self.__monitor_factory is not None:
for entry in all_monitors:
self.__monitors.append(self.__monitor_factory(entry, self.additional_monitor_module_paths))
# Get all of the paths for the logs currently being copied.
all_paths = {}
for entry in self.__logs:
all_paths[entry.log_path] = True
# Now add in a logs entry for each monitor's log file if there is not already
# an entry for it.
for entry in self.__monitors:
log_config = entry.log_config
if type(log_config) is dict:
log_config = JsonObject(content=log_config)
# If the log config does not specify a parser, we add it in.
self.__verify_or_set_optional_string(log_config, 'parser', 'agent-metrics',
'log entry requested by module "%s"' % entry.module_name)
self.__verify_log_entry_and_set_defaults(
log_config, description='log entry requested by module "%s"' % entry.module_name)
path = log_config['path']
# Update the monitor to have the complete log config entry. This also guarantees that the path
# is absolute.
entry.log_config = log_config
if not path in all_paths:
if 'parser' in log_config:
log_config['attributes']['parser'] = log_config['parser']
if self.__log_factory is not None:
self.__logs.append(self.__log_factory(log_config))
all_paths[path] = True
