def __create_test_instance(self, configuration_logs_entry,
monitors_log_configs):
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
logs_json_array = JsonArray()
for entry in configuration_logs_entry:
logs_json_array.add(JsonObject(content=entry))
fp = open(config_file, 'w')
fp.write(
json_lib.serialize(JsonObject(api_key='fake',
logs=logs_json_array)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths)
config.parse()
self.__monitor_fake_instances = []
for monitor_log_config in monitors_log_configs:
self.__monitor_fake_instances.append(
FakeMonitor(monitor_log_config))
# noinspection PyTypeChecker
return CopyingManager(config, self.__monitor_fake_instances)
def test_keys(self):
x = JsonObject(foo="a", bar=10)
keys = list(x.keys())
self.assertEquals(len(keys), 2)
self.assertTrue(keys[0] == "foo" or keys[0] == "bar")
self.assertTrue(keys[1] == "foo" or keys[1] == "bar")
def default_monitors(self):
"""Returns the default monitors to use for this platform.
This is guaranteed to be invoked after consume_config is called to allow implementations to make what they
return be dependent on configuration options.
This method should list of dicts containing monitor configuration options just as you would specify them in
the configuration file. The list may be empty.
@return: The default monitors
@rtype: list<dict>
"""
result = []
if self.__run_system_metrics:
result.append(
JsonObject(
module='scalyr_agent.builtin_monitors.linux_system_metrics'
))
if self.__run_agent_process_metrics:
result.append(
JsonObject(
module=
'scalyr_agent.builtin_monitors.linux_process_metrics',
pid='$$',
id='agent'))
return result
def test_keys(self):
x = JsonObject(foo='a', bar=10)
keys = x.keys()
self.assertEquals(len(keys), 2)
self.assertTrue(keys[0] == 'foo' or keys[0] == 'bar')
self.assertTrue(keys[1] == 'foo' or keys[1] == 'bar')
def create_config_matcher(self, conf):
""" Create a function that will return a log configuration when passed in data that matches that config.
Intended to be overwritten by users of LogConfigManager to match their own use case.
If passed an empty dictionary in `conf` this should create a catchall matcher with default configuration.
@param conf: Logger configuration in the form of a dictionary or JsonObject, that a matcher should be created for.
@return: Logger configuration in the form of a dictionary or JsonObject if this matcher matches the passed
in data, None otherwise
"""
config = copy.deepcopy(conf)
if "journald_unit" not in config:
config["journald_unit"] = ".*"
file_template = Template("journald_${ID}.log")
regex = re.compile(config["journald_unit"])
match_hash = six.text_type(hash(config["journald_unit"]))
if config["journald_unit"] == ".*":
match_hash = "monitor"
full_path = os.path.join(
self._global_config.agent_log_path,
file_template.safe_substitute({"ID": match_hash}),
)
matched_config = JsonObject({"parser": "journald", "path": full_path})
matched_config.update(config)
def config_matcher(unit):
if regex.match(unit) is not None:
return matched_config
return None
return config_matcher
def test_keys(self):
x = JsonObject(foo='a', bar=10)
keys = x.keys()
self.assertEquals(len(keys), 2)
self.assertTrue(keys[0] == 'foo' or keys[0] == 'bar')
self.assertTrue(keys[1] == 'foo' or keys[1] == 'bar')
def __create_test_instance(self, config_monitors, platform_monitors):
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
monitors_json_array = JsonArray()
for entry in config_monitors:
monitors_json_array.add(JsonObject(content=entry))
fp = open(config_file, 'w')
fp.write(
json_lib.serialize(
JsonObject(api_key='fake', monitors=monitors_json_array)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths)
config.parse()
# noinspection PyTypeChecker
return MonitorsManager(config, FakePlatform(platform_monitors))
def test_get_bool(self):
x = JsonObject(foo=True, bar=False)
self.assertEquals(x.get_bool("foo"), True)
self.assertEquals(x.get_bool("bar"), False)
# Test conversion from int to bool
x = JsonObject(foo=1, bar=0)
self.assertEquals(x.get_bool("foo"), True)
self.assertEquals(x.get_bool("bar"), False)
# Test conversion from string to bool
x = JsonObject(foo="ok", bar="f", barb="false", barc="")
self.assertEquals(x.get_bool("foo"), True)
self.assertEquals(x.get_bool("bar"), False)
self.assertEquals(x.get_bool("barb"), False)
self.assertEquals(x.get_bool("barc"), False)
# Test that bad numbers raise an exception
x = JsonObject(foo=5)
self.assertRaises(JsonConversionException, x.get_bool, "foo")
# Test the default value is returned if field is missing.
self.assertEquals(x.get_bool("none", default_value=True), True)
# Test returns none if missing.
self.assertEquals(x.get_bool("none", none_if_missing=True), None)
# Raise an exception when field is missing.
self.assertRaises(JsonMissingFieldException, x.get_bool, "none")
def _initialize(self):
"""Performs monitor-specific initialization."""
# Set up tags for this file.
tags = self._config.get('tags', default=JsonObject())
if not type(tags) is dict and not type(tags) is JsonObject:
raise BadMonitorConfiguration('The configuration field \'tags\' is not a dict or JsonObject', 'tags')
# Make a copy just to be safe.
tags = JsonObject(content=tags)
tags['parser'] = 'agent-metrics'
self.log_config = {
'attributes': tags,
'parser': 'agent-metrics',
'path': 'linux_system_metrics.log',
}
collector_directory = self._config.get('collectors_directory', convert_to=str,
default=SystemMetricsMonitor.__get_collectors_directory())
collector_directory = os.path.realpath(collector_directory)
if not os.path.isdir(collector_directory):
raise BadMonitorConfiguration('No such directory for collectors: %s' % collector_directory,
'collectors_directory')
self.options = TcollectorOptions()
self.options.cdir = collector_directory
self.modules = tcollector.load_etc_dir(self.options, tags)
self.tags = tags
def get_default_monitors(self, config):
"""Returns the default monitors to use for this platform.
This method should return a list of dicts containing monitor configuration options just as you would specify
them in the configuration file. The list may be empty.
@param config The configuration object to use.
@type config configuration.Configuration
@return: The default monitors
@rtype: list<dict>
"""
result = []
if config.implicit_metric_monitor:
result.append(
JsonObject(
module=
"scalyr_agent.builtin_monitors.windows_system_metrics"))
if config.implicit_agent_process_metrics_monitor:
result.append(
JsonObject(
module=
"scalyr_agent.builtin_monitors.windows_process_metrics",
pid="$$",
id="agent",
))
return result
def test_equality(self):
x = JsonObject(foo='a', bar=10)
y = JsonObject(foo='a', bar=10)
z = JsonObject(foo='a', bar=10, zar=True)
self.assertEquals(x, y)
self.assertNotEquals(x, z)
self.assertNotEquals(y, z)
def _create_add_events_request(self, session_info=None, max_size=None):
# Need to override this to return an AddEventsRequest even though we don't have a real scalyr client instance.
if session_info is None:
body = JsonObject(server_attributes=session_info, token='fake')
else:
body = JsonObject(token='fake')
return AddEventsRequest(body, max_size=max_size)
def _initialize(self):
"""Performs monitor-specific initialization."""
# Set up tags for this file.
tags = self._config.get("tags", default=JsonObject())
if type(tags) is not dict and type(tags) is not JsonObject:
raise BadMonitorConfiguration(
"The configuration field 'tags' is not a dict or JsonObject", "tags"
)
# Make a copy just to be safe.
tags = JsonObject(content=tags)
tags["parser"] = "agent-metrics"
self.log_config = {
"attributes": tags,
"parser": "agent-metrics",
"path": "linux_system_metrics.log",
}
collector_directory = self._config.get(
"collectors_directory",
convert_to=six.text_type,
default=SystemMetricsMonitor.__get_collectors_directory(),
)
collector_directory = os.path.realpath(collector_directory)
if not os.path.isdir(collector_directory):
raise BadMonitorConfiguration(
"No such directory for collectors: %s" % collector_directory,
"collectors_directory",
)
self.options = TcollectorOptions()
self.options.cdir = collector_directory
self.options.network_interface_prefixes = self._config.get(
"network_interface_prefixes", default="eth"
)
if isinstance(self.options.network_interface_prefixes, six.string_types):
self.options.network_interface_prefixes = [
self.options.network_interface_prefixes
]
self.options.network_interface_suffix = self._config.get(
"network_interface_suffix", default="[0-9A-Z]+"
)
self.options.local_disks_only = self._config.get("local_disks_only")
self.modules = tcollector.load_etc_dir(self.options, tags)
self.tags = tags
def get_default_monitors(self, config): # type: (Configuration) -> List
"""Returns the default monitors to use for this platform.
This method should return a list of dicts containing monitor configuration options just as you would specify
them in the configuration file. The list may be empty.
@param config The configuration object to use.
@type config configuration.Configuration
@return: The default monitors
@rtype: list<dict>
"""
result = []
if config.implicit_metric_monitor:
result.append(
JsonObject(
module="scalyr_agent.builtin_monitors.linux_system_metrics",
))
if config.implicit_agent_process_metrics_monitor:
result.append(
JsonObject(
module=
"scalyr_agent.builtin_monitors.linux_process_metrics",
pid="$$",
id="agent",
))
# if multi-process workers are enabled and worker session processes monitoring is enabled,
# then create linux metrics monitor for each worker process.
if (config.use_multiprocess_workers
and config.enable_worker_session_process_metrics_gather):
for worker_config in config.worker_configs:
for worker_session_id in config.get_session_ids_of_the_worker(
worker_config):
result.append(
JsonObject(
module=
"scalyr_agent.builtin_monitors.linux_process_metrics",
# the copying manager start after the declaration of the managers,
# so we can not put the real PID but just mark that it will be set later.
pid="$$TBD",
id="{0}{1}".format(
WORKER_SESSION_PROCESS_MONITOR_ID_PREFIX,
worker_session_id,
),
))
return result
def query_all_objects(self):
"""
Queries the k8s api for all objects of a specific type (determined by subclasses).
@return - a JsonObject containing at least one element called 'items' which is a JsonArray of JsonObjects
returned by the query
"""
return JsonObject({'items': JsonArray([])})
def test_default_match(self):
self._write_file_with_separator_conversion(""" {
api_key: "hi there",
k8s_logs: [
{
attributes: {
"foo": "bar",
"baz": "boo",
},
testTest: "test",
}
]
}
""")
config = self._create_test_configuration_instance()
config.parse()
builder = K8sConfigBuilder(config.k8s_log_configs,
self.logger,
rename_no_original=True)
config = builder.get_log_config(
self.info,
self.k8s_info,
self.parser,
)
self.assertEqual(self.parser, config["parser"])
self.assertEqual(self.info["log_path"], config["path"])
self.assertEqual("json", config["parse_format"])
expected = JsonObject({"baz": "boo", "foo": "bar"})
self.assertEqual(expected, config["attributes"])
self.assertEqual("test", config["testTest"])
def test_atomic_write_dict_as_json_file(self):
info = {'a': "hi"}
scalyr_util.atomic_write_dict_as_json_file(self.__path,
self.__path + '~', info)
json_object = scalyr_util.read_file_as_json(self.__path)
self.assertEquals(json_object, JsonObject(a='hi'))
def create_configuration(extra_toplevel_config=None):
"""Creates a blank configuration file with default values. Optionally overwrites top-level key/values.
Sets api_key to 'fake' unless defined in extra_toplevel_config
@param extra_toplevel_config: Dict of top-level key/value objects to overwrite.
@return: The configuration object
@rtype: Configuration
"""
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
toplevel_config = {'api_key': 'fake'}
if extra_toplevel_config:
toplevel_config.update(extra_toplevel_config)
fp = open(config_file, 'w')
fp.write(json_lib.serialize(JsonObject(**toplevel_config)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths, None)
config.parse()
# we need to delete the config dir when done
atexit.register(shutil.rmtree, config_dir)
return config
def query_daemonset(self, namespace, name):
"""Wrapper to query a daemonset in a namespace"""
if not name or not namespace:
return JsonObject()
query = '/apis/apps/v1/namespaces/%s/daemonsets/%s' % (namespace, name)
return self.query_api(query)
def test_get_json_object(self):
y = JsonObject(foo=True)
x = JsonArray(y, "Not an object")
self.assertEquals(len(x), 2)
self.assertTrue(x.get_json_object(0) == y)
self.assertRaises(JsonConversionException,
x.get_json_object, 1)
def query_pod(self, namespace, pod):
"""Wrapper to query a pod in a namespace"""
if not pod or not namespace:
return JsonObject()
query = '/api/v1/namespaces/%s/pods/%s' % (namespace, pod)
return self.query_api(query)
def load_checkpoints(filename):
"""
Atomically loads checkpoints from a file. The checkpoints are only ever loaded from disk once,
and any future calls to this function return the in-memory checkpoints of the first successfully completed call.
@param filename: the path on disk to a JSON file to load checkpoints from
"""
result = None
_global_lock.acquire()
try:
if filename in _global_checkpoints:
result = _global_checkpoints[filename]
finally:
_global_lock.release()
# if checkpoints already exist for this file, return the in memory copy
if result is not None:
return result
# read from the file on disk
checkpoints = JsonObject({})
try:
checkpoints = scalyr_util.read_file_as_json(filename)
except:
global_log.log(
scalyr_logging.DEBUG_LEVEL_1,
"No checkpoint file '%s' exists.\n\tAll journald logs for '%s' will be read starting from their current end.",
filename,
)
checkpoints = JsonObject({})
_global_lock.acquire()
try:
# check if another thread created Checkpoints for this file
# while we were loading from disk and if so, return
# the in memory copy
if filename in _global_checkpoints:
result = _global_checkpoints[filename]
else:
# checkpoints for this file haven't been created yet, so
# create them and store them in the global checkpoints dict
result = Checkpoint(filename, checkpoints)
_global_checkpoints[filename] = result
finally:
_global_lock.release()
return result
def query_object(self, namespace, name):
"""
Queries the k8s api for a single object of a specific type (determined by subclasses).
@param: namespace - the namespace to query in
@param: name - the name of the object
@return - a JsonObject returned by the query
"""
return JsonObject({})
def test_get_float(self):
x = JsonObject(foo=5.2, bar=True)
self.assertEquals(x.get_float("foo"), 5.2)
x = JsonObject(foo="5.2", bar=True)
self.assertEquals(x.get_float("foo"), 5.2)
# Test that bad strings raise an exception
x = JsonObject(foo="fhi3")
self.assertRaises(JsonConversionException, x.get_float, "foo")
# Test the default value is returned if field is missing.
self.assertEquals(x.get_long("none", default_value=5.2), 5.2)
# Test returns none if missing.
self.assertEquals(x.get_long("none", none_if_missing=True), None)
# Raise an exception when field is missing.
self.assertRaises(JsonMissingFieldException, x.get_long, "none")
def test_json_objects(self):
y = JsonObject(foo=True)
x = JsonArray(y)
z = []
for element in x.json_objects():
z.append(element)
self.assertEquals(len(z), 1)
self.assertTrue(x[0] == z[0])
def test_iter(self):
x = JsonObject(foo='a', bar=10)
keys = []
for key in x:
keys.append(key)
self.assertEquals(len(keys), 2)
self.assertTrue('foo' in keys)
self.assertTrue('bar' in keys)
def test_iter(self):
x = JsonObject(foo="a", bar=10)
keys = []
for key in x:
keys.append(key)
self.assertEquals(len(keys), 2)
self.assertTrue("foo" in keys)
self.assertTrue("bar" in keys)
def initialize(self):
"""Generate the config matchers for this manager from the global config"""
config_matchers = []
for config in self._global_config.journald_log_configs:
config_matcher = self.create_config_matcher(config)
config_matchers.append(config_matcher)
# Add a catchall matcher at the end in case one was not configured
config_matchers.append(self.create_config_matcher(JsonObject()))
return config_matchers
def test_get_json_object(self):
x = JsonObject(foo=5, bar=True)
y = JsonObject(bar=x)
self.assertTrue(y.get_json_object("bar") == x)
# Test the default value is returned if field is missing.
self.assertTrue(x.get_json_object("none", default_value=x) == x)
# Test returns none if missing.
self.assertEquals(x.get_json_object("none", none_if_missing=True), None)
# Raise an exception when field is missing.
self.assertRaises(JsonMissingFieldException,
y.get_json_object, "none")
# Raise an exception if field is not JsonObject
self.assertRaises(JsonConversionException,
x.get_json_object, "foo")
def __create_test_instance(self, use_pipelining=False):
tmp_dir = tempfile.mkdtemp()
config_dir = os.path.join(tmp_dir, "config")
data_dir = os.path.join(tmp_dir, "data")
log_dir = os.path.join(tmp_dir, "log")
os.mkdir(data_dir)
os.mkdir(config_dir)
os.mkdir(log_dir)
self.__test_log_file = os.path.join(tmp_dir, "test.log")
fp = open(self.__test_log_file, "w")
fp.close()
config_file = os.path.join(config_dir, "agentConfig.json")
config_fragments_dir = os.path.join(config_dir, "configs.d")
os.makedirs(config_fragments_dir)
logs_json_array = JsonArray()
logs_json_array.add(JsonObject(path=self.__test_log_file))
pipeline_threshold = 1.1
if use_pipelining:
pipeline_threshold = 0.0
fp = open(config_file, "w")
fp.write(
json_lib.serialize(
JsonObject(
api_key="fake",
logs=logs_json_array,
pipeline_threshold=pipeline_threshold,
)))
fp.close()
default_paths = DefaultPaths(log_dir, config_file, data_dir)
config = Configuration(config_file, default_paths, None)
config.parse()
# noinspection PyTypeChecker
self._controller = TestableCopyingManager(config, []).controller
return self._controller
def test_iter(self):
y = JsonObject(foo=True)
x = JsonArray(y, "Not an object")
z = []
for element in x:
z.append(element)
self.assertEquals(len(z), 2)
self.assertTrue(x[0] == z[0])
self.assertTrue(x[1] == z[1])
def __get_docker_logs(self, containers):
result = []
attributes = None
try:
attributes = JsonObject({"monitor": "agentDocker"})
if self.__host_hostname:
attributes['serverHost'] = self.__host_hostname
except Exception, e:
self._logger.error(
"Error setting monitor attribute in DockerMonitor")
raise
def __create_test_configuration_instance(self):
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
def log_factory(config):
return CopyingParamsTest.LogObject(config)
def monitor_factory(config, _):
return CopyingParamsTest.MonitorObject(config)
monitors = [
JsonObject(
module='scalyr_agent.builtin_monitors.linux_system_metrics'),
JsonObject(
module='scalyr_agent.builtin_monitors.linux_process_metrics',
pid='$$',
id='agent')
]
return Configuration(self.__config_file, default_paths, monitors,
log_factory, monitor_factory)
def test_get_string(self):
x = JsonObject(foo="hi")
self.assertEquals(x.get_string("foo"), "hi")
x = JsonObject(foo=1)
self.assertEquals(x.get_string("foo"), "1")
# Test the default value is returned if field is missing.
self.assertEquals(x.get_string("none", default_value="ok"), "ok")
# Test returns none if missing.
self.assertEquals(x.get_string("none", none_if_missing=True), None)
# Raise an exception when field is missing.
self.assertRaises(JsonMissingFieldException, x.get_string, "none")
def test_get_float(self):
x = JsonObject(foo=5.2, bar=True)
self.assertEquals(x.get_float("foo"), 5.2)
x = JsonObject(foo="5.2", bar=True)
self.assertEquals(x.get_float("foo"), 5.2)
# Test that bad strings raise an exception
x = JsonObject(foo="fhi3")
self.assertRaises(JsonConversionException, x.get_float, "foo")
# Test the default value is returned if field is missing.
self.assertEquals(x.get_long("none", default_value=5.2), 5.2)
# Test returns none if missing.
self.assertEquals(x.get_long("none", none_if_missing=True), None)
# Raise an exception when field is missing.
self.assertRaises(JsonMissingFieldException, x.get_long, "none")
def __parse_object(self):
"""Parse a JSON object. The scanner must be at the first '{'."""
object_start = self.__scanner.position
self.__scanner.read_ubyte()
result_object = JsonObject()
while True:
c = self.__peek_next_non_whitespace()
if c is None:
return self.__error("Need '}' for end of object", object_start)
elif c == '"':
key = self.__parse_string()
elif c == '_' or 'a' <= c <= 'z' or 'A' <= c <= 'Z':
key = self.__parse_identifier()
next_char = self.__scanner.peek_next_ubyte(
none_if_bad_index=True)
if next_char is None:
return self.__error("Need '}' for end of object",
object_start)
if ord(next_char) > 32 and next_char != ':':
self.__error("To use character '%s' in an attribute name, "
"you must place the attribute name in "
"double-quotes" % next_char)
elif c == '}':
# End-of-object.
self.__scanner.read_ubyte()
return result_object
else:
return self.__error(
"Expected string literal for object attribute name")
self.__peek_next_non_whitespace()
c = self.__scanner.read_ubyte()
if c != ':':
self.__error("Expected ':' delimiting object attribute value")
# skip any whitespace after the colon
self.__peek_next_non_whitespace()
if self.check_duplicate_keys and result_object.__contains__(key):
self.__error("Duplicate key [" + key + "]", object_start)
result_object.put(key, self.parse_value())
c = self.__peek_next_non_whitespace()
if c is None:
self.__error("Need '}' for end of object", object_start)
elif c == '}':
# do nothing we'll process the '}' back around at the top of
# the loop.
continue
elif c == ',':
self.__scanner.read_ubyte()
else:
if self.__preceding_line_break() and self.allow_missing_commas:
# proceed, inferring a comma
continue
else:
self.__error("After object field, expected ',' or '}' but "
"found '%s'... are you missing a comma?" % c)
def test_constructor(self):
x = JsonObject(foo=5, bar=True)
self.assertEquals(x["foo"], 5)
self.assertEquals(x.get("bar"), True)
def parse(self):
self.__read_time = time.time()
try:
try:
# First read the file. This makes sure it exists and can be parsed.
self.__config = scalyr_util.read_file_as_json(self.__file_path)
# What implicit entries do we need to add? metric monitor, agent.log, and then logs from all monitors.
except JsonReadFileException, e:
raise BadConfiguration(str(e), None, 'fileParseError')
# Import any requested variables from the shell and use them for substitutions.
self.__import_shell_variables()
self.__perform_substitutions()
self.__verify_main_config_and_apply_defaults(self.__config, self.__file_path)
self.__verify_logs_and_monitors_configs_and_apply_defaults(self.__config, self.__file_path)
# Now, look for any additional configuration in the config fragment directory.
for fp in self.__list_files(self.config_directory):
self.__additional_paths.append(fp)
content = scalyr_util.read_file_as_json(fp)
for k in content.keys():
if k not in ('logs', 'monitors', 'server_attributes'):
self.__last_error = BadConfiguration(
'Configuration fragment file "%s" contains an invalid key "%s". The config files in the '
'configuration directory can only contain "logs", "monitors", and "server_attributes" '
'entries.' % (fp, k), k, 'badFragmentKey')
raise self.__last_error
self.__verify_logs_and_monitors_configs_and_apply_defaults(content, fp)
self.__add_elements_from_array('logs', content, self.__config)
self.__add_elements_from_array('monitors', content, self.__config)
self.__merge_server_attributes(fp, content, self.__config)
# Add in 'serverHost' to server_attributes if it is not set. We must do this after merging any
# server attributes from the config fragments.
if 'serverHost' not in self.server_attributes:
self.__config['server_attributes']['serverHost'] = socket.gethostname()
# Add in implicit entry to collect the log generated by this agent.
agent_log = None
if self.implicit_agent_log_collection:
config = JsonObject(path='agent.log')
self.__verify_log_entry_and_set_defaults(config, description='implicit rule')
agent_log = config
# Add in any platform-specific monitors.
platform_monitors = []
for monitor in self.__default_monitors:
config = JsonObject(content=monitor)
self.__verify_monitor_entry_and_set_defaults(config, 'default monitors for platform', -1)
platform_monitors.append(config)
all_logs = list(self.__config.get_json_array('logs'))
if agent_log is not None:
all_logs.append(agent_log)
# We need to go back and fill in the monitor id if it is not set. We do this by keeping a count of
# how many monitors we have with the same module name (just considering the last element of the module
# path). We use the shortened form of the module name because that is used when emitting lines for
# this monitor in the logs -- see scalyr_logging.py.
monitors_by_module_name = {}
# Tracks which modules already had an id present in the module config.
had_id = {}
all_monitors = list(self.__config.get_json_array('monitors'))
for monitor in platform_monitors:
all_monitors.append(monitor)
for entry in all_monitors:
module_name = entry['module'].split('.')[-1]
if not module_name in monitors_by_module_name:
index = 1
else:
index = monitors_by_module_name[module_name] + 1
if 'id' not in entry:
entry['id'] = index
else:
had_id[module_name] = True
monitors_by_module_name[module_name] = index
# Just as a simplification, if there is only one monitor with a given name, we remove the monitor_id
# to clean up it's name in the logs.
for entry in all_monitors:
module_name = entry['module'].split('.')[-1]
if monitors_by_module_name[module_name] == 1 and not module_name in had_id:
entry['id'] = ''
# Now build up __logs to have an object created for each log entry, and __monitors to have an object
# created for each monitor entry.
for entry in all_logs:
# Automatically add in the parser to the attributes section. We make a copy of the object first
# just to be safe.
entry = JsonObject(content=entry)
if 'parser' in entry:
entry['attributes']['parser'] = entry['parser']
if self.__log_factory is not None:
self.__logs.append(self.__log_factory(entry))
if self.__monitor_factory is not None:
for entry in all_monitors:
self.__monitors.append(self.__monitor_factory(entry, self.additional_monitor_module_paths))
# Get all of the paths for the logs currently being copied.
all_paths = {}
for entry in self.__logs:
all_paths[entry.log_path] = True
# Now add in a logs entry for each monitor's log file if there is not already
# an entry for it.
for entry in self.__monitors:
log_config = entry.log_config
if type(log_config) is dict:
log_config = JsonObject(content=log_config)
# If the log config does not specify a parser, we add it in.
self.__verify_or_set_optional_string(log_config, 'parser', 'agent-metrics',
'log entry requested by module "%s"' % entry.module_name)
self.__verify_log_entry_and_set_defaults(
log_config, description='log entry requested by module "%s"' % entry.module_name)
path = log_config['path']
# Update the monitor to have the complete log config entry. This also guarantees that the path
# is absolute.
entry.log_config = log_config
if not path in all_paths:
if 'parser' in log_config:
log_config['attributes']['parser'] = log_config['parser']
if self.__log_factory is not None:
self.__logs.append(self.__log_factory(log_config))
all_paths[path] = True
def test_get_or_create_json_object(self):
x = JsonObject(foo=5, bar=True)
y = JsonObject(bar=x)
self.assertTrue(y.get_or_create_json_object("bar") == x)
self.assertEquals(len(y.get_or_create_json_object("foo")), 0)
