def __create_test_instance(self, configuration_logs_entry,
monitors_log_configs):
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
logs_json_array = JsonArray()
for entry in configuration_logs_entry:
logs_json_array.add(JsonObject(content=entry))
fp = open(config_file, 'w')
fp.write(
json_lib.serialize(JsonObject(api_key='fake',
logs=logs_json_array)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths)
config.parse()
self.__monitor_fake_instances = []
for monitor_log_config in monitors_log_configs:
self.__monitor_fake_instances.append(
FakeMonitor(monitor_log_config))
# noinspection PyTypeChecker
return CopyingManager(config, self.__monitor_fake_instances)
def _initialize(self):
"""Performs monitor-specific initialization."""
# Set up tags for this file.
tags = self._config.get('tags', default=JsonObject())
if not type(tags) is dict and not type(tags) is JsonObject:
raise BadMonitorConfiguration('The configuration field \'tags\' is not a dict or JsonObject', 'tags')
# Make a copy just to be safe.
tags = JsonObject(content=tags)
tags['parser'] = 'agent-metrics'
self.log_config = {
'attributes': tags,
'parser': 'agent-metrics',
'path': 'linux_system_metrics.log',
}
collector_directory = self._config.get('collectors_directory', convert_to=str,
default=SystemMetricsMonitor.__get_collectors_directory())
collector_directory = os.path.realpath(collector_directory)
if not os.path.isdir(collector_directory):
raise BadMonitorConfiguration('No such directory for collectors: %s' % collector_directory,
'collectors_directory')
self.options = TcollectorOptions()
self.options.cdir = collector_directory
self.modules = tcollector.load_etc_dir(self.options, tags)
self.tags = tags
def __create_test_instance(self, config_monitors, platform_monitors):
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
monitors_json_array = JsonArray()
for entry in config_monitors:
monitors_json_array.add(JsonObject(content=entry))
fp = open(config_file, 'w')
fp.write(
json_lib.serialize(
JsonObject(api_key='fake', monitors=monitors_json_array)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths)
config.parse()
# noinspection PyTypeChecker
return MonitorsManager(config, FakePlatform(platform_monitors))
def test_get_bool(self):
x = JsonObject(foo=True, bar=False)
self.assertEquals(x.get_bool("foo"), True)
self.assertEquals(x.get_bool("bar"), False)
# Test conversion from int to bool
x = JsonObject(foo=1, bar=0)
self.assertEquals(x.get_bool("foo"), True)
self.assertEquals(x.get_bool("bar"), False)
# Test conversion from string to bool
x = JsonObject(foo="ok", bar="f", barb="false", barc="")
self.assertEquals(x.get_bool("foo"), True)
self.assertEquals(x.get_bool("bar"), False)
self.assertEquals(x.get_bool("barb"), False)
self.assertEquals(x.get_bool("barc"), False)
# Test that bad numbers raise an exception
x = JsonObject(foo=5)
self.assertRaises(JsonConversionException, x.get_bool, "foo")
# Test the default value is returned if field is missing.
self.assertEquals(x.get_bool("none", default_value=True), True)
# Test returns none if missing.
self.assertEquals(x.get_bool("none", none_if_missing=True), None)
# Raise an exception when field is missing.
self.assertRaises(JsonMissingFieldException, x.get_bool, "none")
def default_monitors(self):
"""Returns the default monitors to use for this platform.
This is guaranteed to be invoked after consume_config is called to allow implementations to make what they
return be dependent on configuration options.
This method should list of dicts containing monitor configuration options just as you would specify them in
the configuration file. The list may be empty.
@return: The default monitors
@rtype: list<dict>
"""
result = []
if self.__run_system_metrics:
result.append(
JsonObject(
module='scalyr_agent.builtin_monitors.linux_system_metrics'
))
if self.__run_agent_process_metrics:
result.append(
JsonObject(
module=
'scalyr_agent.builtin_monitors.linux_process_metrics',
pid='$$',
id='agent'))
return result
def get_default_monitors(self, config):
"""Returns the default monitors to use for this platform.
This method should return a list of dicts containing monitor configuration options just as you would specify
them in the configuration file. The list may be empty.
@param config The configuration object to use.
@type config configuration.Configuration
@return: The default monitors
@rtype: list<dict>
"""
result = []
if config.implicit_metric_monitor:
result.append(
JsonObject(
module=
"scalyr_agent.builtin_monitors.windows_system_metrics"))
if config.implicit_agent_process_metrics_monitor:
result.append(
JsonObject(
module=
"scalyr_agent.builtin_monitors.windows_process_metrics",
pid="$$",
id="agent",
))
return result
def _create_add_events_request(self, session_info=None, max_size=None):
# Need to override this to return an AddEventsRequest even though we don't have a real scalyr client instance.
if session_info is None:
body = JsonObject(server_attributes=session_info, token='fake')
else:
body = JsonObject(token='fake')
return AddEventsRequest(body, max_size=max_size)
def test_equality(self):
x = JsonObject(foo='a', bar=10)
y = JsonObject(foo='a', bar=10)
z = JsonObject(foo='a', bar=10, zar=True)
self.assertEquals(x, y)
self.assertNotEquals(x, z)
self.assertNotEquals(y, z)
def _initialize(self):
"""Performs monitor-specific initialization."""
# Set up tags for this file.
tags = self._config.get("tags", default=JsonObject())
if type(tags) is not dict and type(tags) is not JsonObject:
raise BadMonitorConfiguration(
"The configuration field 'tags' is not a dict or JsonObject", "tags"
)
# Make a copy just to be safe.
tags = JsonObject(content=tags)
tags["parser"] = "agent-metrics"
self.log_config = {
"attributes": tags,
"parser": "agent-metrics",
"path": "linux_system_metrics.log",
}
collector_directory = self._config.get(
"collectors_directory",
convert_to=six.text_type,
default=SystemMetricsMonitor.__get_collectors_directory(),
)
collector_directory = os.path.realpath(collector_directory)
if not os.path.isdir(collector_directory):
raise BadMonitorConfiguration(
"No such directory for collectors: %s" % collector_directory,
"collectors_directory",
)
self.options = TcollectorOptions()
self.options.cdir = collector_directory
self.options.network_interface_prefixes = self._config.get(
"network_interface_prefixes", default="eth"
)
if isinstance(self.options.network_interface_prefixes, six.string_types):
self.options.network_interface_prefixes = [
self.options.network_interface_prefixes
]
self.options.network_interface_suffix = self._config.get(
"network_interface_suffix", default="[0-9A-Z]+"
)
self.options.local_disks_only = self._config.get("local_disks_only")
self.modules = tcollector.load_etc_dir(self.options, tags)
self.tags = tags
def test_get_string(self):
x = JsonObject(foo="hi")
self.assertEquals(x.get_string("foo"), "hi")
x = JsonObject(foo=1)
self.assertEquals(x.get_string("foo"), "1")
# Test the default value is returned if field is missing.
self.assertEquals(x.get_string("none", default_value="ok"), "ok")
# Test returns none if missing.
self.assertEquals(x.get_string("none", none_if_missing=True), None)
# Raise an exception when field is missing.
self.assertRaises(JsonMissingFieldException, x.get_string, "none")
def get_default_monitors(self, config): # type: (Configuration) -> List
"""Returns the default monitors to use for this platform.
This method should return a list of dicts containing monitor configuration options just as you would specify
them in the configuration file. The list may be empty.
@param config The configuration object to use.
@type config configuration.Configuration
@return: The default monitors
@rtype: list<dict>
"""
result = []
if config.implicit_metric_monitor:
result.append(
JsonObject(
module="scalyr_agent.builtin_monitors.linux_system_metrics",
))
if config.implicit_agent_process_metrics_monitor:
result.append(
JsonObject(
module=
"scalyr_agent.builtin_monitors.linux_process_metrics",
pid="$$",
id="agent",
))
# if multi-process workers are enabled and worker session processes monitoring is enabled,
# then create linux metrics monitor for each worker process.
if (config.use_multiprocess_workers
and config.enable_worker_session_process_metrics_gather):
for worker_config in config.worker_configs:
for worker_session_id in config.get_session_ids_of_the_worker(
worker_config):
result.append(
JsonObject(
module=
"scalyr_agent.builtin_monitors.linux_process_metrics",
# the copying manager start after the declaration of the managers,
# so we can not put the real PID but just mark that it will be set later.
pid="$$TBD",
id="{0}{1}".format(
WORKER_SESSION_PROCESS_MONITOR_ID_PREFIX,
worker_session_id,
),
))
return result
def test_atomic_write_dict_as_json_file(self):
info = {'a': "hi"}
scalyr_util.atomic_write_dict_as_json_file(self.__path,
self.__path + '~', info)
json_object = scalyr_util.read_file_as_json(self.__path)
self.assertEquals(json_object, JsonObject(a='hi'))
def query_daemonset(self, namespace, name):
"""Wrapper to query a daemonset in a namespace"""
if not name or not namespace:
return JsonObject()
query = '/apis/apps/v1/namespaces/%s/daemonsets/%s' % (namespace, name)
return self.query_api(query)
def create_configuration(extra_toplevel_config=None):
"""Creates a blank configuration file with default values. Optionally overwrites top-level key/values.
Sets api_key to 'fake' unless defined in extra_toplevel_config
@param extra_toplevel_config: Dict of top-level key/value objects to overwrite.
@return: The configuration object
@rtype: Configuration
"""
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
toplevel_config = {'api_key': 'fake'}
if extra_toplevel_config:
toplevel_config.update(extra_toplevel_config)
fp = open(config_file, 'w')
fp.write(json_lib.serialize(JsonObject(**toplevel_config)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths, None)
config.parse()
# we need to delete the config dir when done
atexit.register(shutil.rmtree, config_dir)
return config
def create_config_matcher(self, conf):
""" Create a function that will return a log configuration when passed in data that matches that config.
Intended to be overwritten by users of LogConfigManager to match their own use case.
If passed an empty dictionary in `conf` this should create a catchall matcher with default configuration.
@param conf: Logger configuration in the form of a dictionary or JsonObject, that a matcher should be created for.
@return: Logger configuration in the form of a dictionary or JsonObject if this matcher matches the passed
in data, None otherwise
"""
config = copy.deepcopy(conf)
if "journald_unit" not in config:
config["journald_unit"] = ".*"
file_template = Template("journald_${ID}.log")
regex = re.compile(config["journald_unit"])
match_hash = six.text_type(hash(config["journald_unit"]))
if config["journald_unit"] == ".*":
match_hash = "monitor"
full_path = os.path.join(
self._global_config.agent_log_path,
file_template.safe_substitute({"ID": match_hash}),
)
matched_config = JsonObject({"parser": "journald", "path": full_path})
matched_config.update(config)
def config_matcher(unit):
if regex.match(unit) is not None:
return matched_config
return None
return config_matcher
def test_get_json_object(self):
y = JsonObject(foo=True)
x = JsonArray(y, "Not an object")
self.assertEquals(len(x), 2)
self.assertTrue(x.get_json_object(0) == y)
self.assertRaises(JsonConversionException,
x.get_json_object, 1)
def query_all_objects(self):
"""
Queries the k8s api for all objects of a specific type (determined by subclasses).
@return - a JsonObject containing at least one element called 'items' which is a JsonArray of JsonObjects
returned by the query
"""
return JsonObject({'items': JsonArray([])})
def test_keys(self):
x = JsonObject(foo='a', bar=10)
keys = x.keys()
self.assertEquals(len(keys), 2)
self.assertTrue(keys[0] == 'foo' or keys[0] == 'bar')
self.assertTrue(keys[1] == 'foo' or keys[1] == 'bar')
def test_keys(self):
x = JsonObject(foo="a", bar=10)
keys = list(x.keys())
self.assertEquals(len(keys), 2)
self.assertTrue(keys[0] == "foo" or keys[0] == "bar")
self.assertTrue(keys[1] == "foo" or keys[1] == "bar")
def test_default_match(self):
self._write_file_with_separator_conversion(""" {
api_key: "hi there",
k8s_logs: [
{
attributes: {
"foo": "bar",
"baz": "boo",
},
testTest: "test",
}
]
}
""")
config = self._create_test_configuration_instance()
config.parse()
builder = K8sConfigBuilder(config.k8s_log_configs,
self.logger,
rename_no_original=True)
config = builder.get_log_config(
self.info,
self.k8s_info,
self.parser,
)
self.assertEqual(self.parser, config["parser"])
self.assertEqual(self.info["log_path"], config["path"])
self.assertEqual("json", config["parse_format"])
expected = JsonObject({"baz": "boo", "foo": "bar"})
self.assertEqual(expected, config["attributes"])
self.assertEqual("test", config["testTest"])
def query_pod(self, namespace, pod):
"""Wrapper to query a pod in a namespace"""
if not pod or not namespace:
return JsonObject()
query = '/api/v1/namespaces/%s/pods/%s' % (namespace, pod)
return self.query_api(query)
def load_checkpoints(filename):
"""
Atomically loads checkpoints from a file. The checkpoints are only ever loaded from disk once,
and any future calls to this function return the in-memory checkpoints of the first successfully completed call.
@param filename: the path on disk to a JSON file to load checkpoints from
"""
result = None
_global_lock.acquire()
try:
if filename in _global_checkpoints:
result = _global_checkpoints[filename]
finally:
_global_lock.release()
# if checkpoints already exist for this file, return the in memory copy
if result is not None:
return result
# read from the file on disk
checkpoints = JsonObject({})
try:
checkpoints = scalyr_util.read_file_as_json(filename)
except:
global_log.log(
scalyr_logging.DEBUG_LEVEL_1,
"No checkpoint file '%s' exists.\n\tAll journald logs for '%s' will be read starting from their current end.",
filename,
)
checkpoints = JsonObject({})
_global_lock.acquire()
try:
# check if another thread created Checkpoints for this file
# while we were loading from disk and if so, return
# the in memory copy
if filename in _global_checkpoints:
result = _global_checkpoints[filename]
else:
# checkpoints for this file haven't been created yet, so
# create them and store them in the global checkpoints dict
result = Checkpoint(filename, checkpoints)
_global_checkpoints[filename] = result
finally:
_global_lock.release()
return result
def test_get_json_object(self):
x = JsonObject(foo=5, bar=True)
y = JsonObject(bar=x)
self.assertTrue(y.get_json_object("bar") == x)
# Test the default value is returned if field is missing.
self.assertTrue(x.get_json_object("none", default_value=x) == x)
# Test returns none if missing.
self.assertEquals(x.get_json_object("none", none_if_missing=True), None)
# Raise an exception when field is missing.
self.assertRaises(JsonMissingFieldException, y.get_json_object, "none")
# Raise an exception if field is not JsonObject
self.assertRaises(JsonConversionException, x.get_json_object, "foo")
def query_object(self, namespace, name):
"""
Queries the k8s api for a single object of a specific type (determined by subclasses).
@param: namespace - the namespace to query in
@param: name - the name of the object
@return - a JsonObject returned by the query
"""
return JsonObject({})
def test_json_object_to_dict_memory_leak(self):
content = {"foo": "bar", "a": 2, "b": [1, 2, 3]}
# New object
for index in range(0, 50):
json_object = JsonObject(content=content)
dict_value = json_object.to_dict()
self.assertEqual(content, dict_value)
self.assertNoNewGarbage()
# Existing object
json_object = JsonObject(content=content)
for index in range(0, 50):
dict_value = json_object.to_dict()
self.assertEqual(content, dict_value)
self.assertNoNewGarbage()
def test_json_objects(self):
y = JsonObject(foo=True)
x = JsonArray(y)
z = []
for element in x.json_objects():
z.append(element)
self.assertEquals(len(z), 1)
self.assertTrue(x[0] == z[0])
def test_iter(self):
x = JsonObject(foo='a', bar=10)
keys = []
for key in x:
keys.append(key)
self.assertEquals(len(keys), 2)
self.assertTrue('foo' in keys)
self.assertTrue('bar' in keys)
def initialize(self):
"""Generate the config matchers for this manager from the global config"""
config_matchers = []
for config in self._global_config.journald_log_configs:
config_matcher = self.create_config_matcher(config)
config_matchers.append(config_matcher)
# Add a catchall matcher at the end in case one was not configured
config_matchers.append(self.create_config_matcher(JsonObject()))
return config_matchers
def test_get_float(self):
x = JsonObject(foo=5.2, bar=True)
self.assertEquals(x.get_float("foo"), 5.2)
x = JsonObject(foo="5.2", bar=True)
self.assertEquals(x.get_float("foo"), 5.2)
# Test that bad strings raise an exception
x = JsonObject(foo="fhi3")
self.assertRaises(JsonConversionException, x.get_float, "foo")
# Test the default value is returned if field is missing.
self.assertEquals(x.get_long("none", default_value=5.2), 5.2)
# Test returns none if missing.
self.assertEquals(x.get_long("none", none_if_missing=True), None)
# Raise an exception when field is missing.
self.assertRaises(JsonMissingFieldException, x.get_long, "none")
def test_iter(self):
x = JsonObject(foo="a", bar=10)
keys = []
for key in x:
keys.append(key)
self.assertEquals(len(keys), 2)
self.assertTrue("foo" in keys)
self.assertTrue("bar" in keys)
