def test_equals(self):
x = JsonArray(1, 2)
y = JsonArray(1, 2)
z = JsonArray(3, 4)
self.assertEquals(x, y)
self.assertNotEqual(x, z)
self.assertNotEqual(y, z)
def test_get_json_object(self):
y = JsonObject(foo=True)
x = JsonArray(y, "Not an object")
self.assertEquals(len(x), 2)
self.assertTrue(x.get_json_object(0) == y)
self.assertRaises(JsonConversionException,
x.get_json_object, 1)
def test_get_server_attribute_no_override(self):
logs_json_array = JsonArray()
config = ScalyrTestUtils.create_configuration(
extra_toplevel_config={'logs': logs_json_array})
self.__monitor_fake_instances = []
monitor_a = FakeMonitor1({'path': 'testA.log'},
id='a',
attribute_key='common_key')
monitor_b = FakeMonitor1({'path': 'testB.log'},
id='b',
attribute_key='common_key')
self.__monitor_fake_instances.append(monitor_a)
self.__monitor_fake_instances.append(monitor_b)
copy_manager = CopyingManager(config, self.__monitor_fake_instances)
if monitor_a.access_order < monitor_b.access_order:
first_accessed = monitor_a
second_accessed = monitor_b
else:
first_accessed = monitor_b
second_accessed = monitor_a
self.assertLess(first_accessed.access_order,
second_accessed.access_order)
self.assertEquals(
copy_manager.expanded_server_attributes['common_key'],
first_accessed.attribute_value)
def test_import_vars_in_configuration_directory(self):
os.environ['TEST_VAR'] = 'bye'
self.__write_file_with_separator_conversion(""" { api_key: "hi there"
logs: [ { path:"/var/log/tomcat6/access.log" }],
server_attributes: { serverHost:"foo.com" }
}
""")
self.__write_config_fragment_file_with_separator_conversion('nginx.json', """ {
import_vars: [ "TEST_VAR" ],
logs: [ { path: "/var/log/nginx/$TEST_VAR.log" } ],
server_attributes: { webServer:"true"}
}
""")
config = self.__create_test_configuration_instance()
config.parse()
self.assertEquals(len(config.additional_file_paths), 1)
additional_paths = list(config.additional_file_paths)
additional_paths.sort()
self.assertTrue(additional_paths[0].endswith('nginx.json'))
self.assertEquals(len(config.log_configs), 3)
self.assertPathEquals(config.log_configs[0].get_string('path'), '/var/log/tomcat6/access.log')
self.assertPathEquals(config.log_configs[1].get_string('path'), '/var/log/nginx/bye.log')
self.assertEquals(config.log_configs[0].get_json_array('sampling_rules'), JsonArray())
self.assertEquals(config.server_attributes['webServer'], 'true')
self.assertEquals(config.server_attributes['serverHost'], 'foo.com')
def __create_test_instance(self, config_monitors, platform_monitors):
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
monitors_json_array = JsonArray()
for entry in config_monitors:
monitors_json_array.add(JsonObject(content=entry))
fp = open(config_file, 'w')
fp.write(
json_lib.serialize(
JsonObject(api_key='fake', monitors=monitors_json_array)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths)
config.parse()
# noinspection PyTypeChecker
return MonitorsManager(config, FakePlatform(platform_monitors))
def __create_test_instance(self, configuration_logs_entry,
monitors_log_configs):
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
logs_json_array = JsonArray()
for entry in configuration_logs_entry:
logs_json_array.add(JsonObject(content=entry))
fp = open(config_file, 'w')
fp.write(
json_lib.serialize(JsonObject(api_key='fake',
logs=logs_json_array)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths)
config.parse()
self.__monitor_fake_instances = []
for monitor_log_config in monitors_log_configs:
self.__monitor_fake_instances.append(
FakeMonitor(monitor_log_config))
# noinspection PyTypeChecker
return CopyingManager(config, self.__monitor_fake_instances)
def test_json_objects(self):
y = JsonObject(foo=True)
x = JsonArray(y)
z = []
for element in x.json_objects():
z.append(element)
self.assertEquals(len(z), 1)
self.assertTrue(x[0] == z[0])
def test_iter(self):
y = JsonObject(foo=True)
x = JsonArray(y, "Not an object")
z = []
for element in x:
z.append(element)
self.assertEquals(len(z), 2)
self.assertTrue(x[0] == z[0])
self.assertTrue(x[1] == z[1])
def __create_test_instance(self, configuration_logs_entry, monitors_log_configs):
logs_json_array = JsonArray()
for entry in configuration_logs_entry:
logs_json_array.add(JsonObject(content=entry))
config = ScalyrTestUtils.create_configuration(extra_toplevel_config={'logs': logs_json_array})
self.__monitor_fake_instances = []
for monitor_log_config in monitors_log_configs:
self.__monitor_fake_instances.append(FakeMonitor(monitor_log_config))
# noinspection PyTypeChecker
return CopyingManager(config, self.__monitor_fake_instances)
def create_test_monitors_manager(
config_monitors=None,
platform_monitors=None,
extra_toplevel_config=None,
null_logger=False,
fake_clock=False,
):
"""Create a test MonitorsManager
@param config_monitors: config monitors
@param platform_monitors: platform monitors
@param extra_toplevel_config: dict of extra top-level key value objects
@param null_logger: If True, set all monitors to log to Nullhandler
@param fake_clock: If non-null, the manager and all it's monitors' _run_state's will use the provided fake_clock
"""
monitors_json_array = JsonArray()
if config_monitors:
for entry in config_monitors:
monitors_json_array.add(JsonObject(content=entry))
extras = {"monitors": monitors_json_array}
if extra_toplevel_config:
extras.update(extra_toplevel_config)
config = ScalyrTestUtils.create_configuration(
extra_toplevel_config=extras)
config.parse()
if not platform_monitors:
platform_monitors = []
# noinspection PyTypeChecker
test_manager = MonitorsManager(config, FakePlatform(platform_monitors))
if null_logger:
# Override Agent Logger to prevent writing to disk
for monitor in test_manager.monitors:
monitor._logger = FakeAgentLogger("fake_agent_logger")
if fake_clock:
for monitor in test_manager.monitors + [test_manager]:
monitor._run_state = scalyr_util.RunState(
fake_clock=fake_clock)
# AGENT-113 set all monitors and the monitors_manager threads to daemon to eliminate occasionally hanging tests
test_manager.setDaemon(True)
for monitor in test_manager.monitors:
if isinstance(monitor, threading.Thread):
monitor.setDaemon(True)
return test_manager, config
def test_to_dict(self):
x = JsonObject(foo="a", bar=10)
self.assertEquals(dict(foo="a", bar=10), x.to_dict())
x = JsonObject(foo=JsonObject(bee=1), bar=10)
self.assertEquals(dict(foo=dict(bee=1), bar=10), x.to_dict())
x = JsonObject(foo=dict(bee=1, boo=JsonObject(hi=True)), bar=10)
self.assertEquals(dict(foo=dict(bee=1, boo=dict(hi=True)), bar=10),
x.to_dict())
x = JsonObject(foo=JsonArray(1, 2, 3), bar=10)
self.assertEquals(dict(foo=[1, 2, 3], bar=10), x.to_dict())
x = JsonObject(foo=[1, 2, JsonObject(foo=5)], bar=10)
self.assertEquals(dict(foo=[1, 2, dict(foo=5)], bar=10), x.to_dict())
def test_get_server_attribute(self):
logs_json_array = JsonArray()
config = ScalyrTestUtils.create_configuration(
extra_toplevel_config={'logs': logs_json_array})
self.__monitor_fake_instances = []
monitor_a = FakeMonitor1({'path': 'testA.log'},
id='a',
attribute_key='KEY_a')
monitor_b = FakeMonitor1({'path': 'testB.log'},
id='b',
attribute_key='KEY_b')
self.__monitor_fake_instances.append(monitor_a)
self.__monitor_fake_instances.append(monitor_b)
copy_manager = CopyingManager(config, self.__monitor_fake_instances)
attribs = copy_manager.expanded_server_attributes
self.assertEquals(attribs['KEY_a'], monitor_a.attribute_value)
self.assertEquals(attribs['KEY_b'], monitor_b.attribute_value)
def test_get_server_attribute(self):
logs_json_array = JsonArray()
config = ScalyrTestUtils.create_configuration(
extra_toplevel_config={"logs": logs_json_array})
self.__monitor_fake_instances = []
monitor_a = FakeMonitor1({"path": "testA.log"},
id="a",
attribute_key="KEY_a")
monitor_b = FakeMonitor1({"path": "testB.log"},
id="b",
attribute_key="KEY_b")
self.__monitor_fake_instances.append(monitor_a)
self.__monitor_fake_instances.append(monitor_b)
copy_manager = CopyingManager(config, self.__monitor_fake_instances)
attribs = copy_manager.expanded_server_attributes
self.assertEquals(attribs["KEY_a"], monitor_a.attribute_value)
self.assertEquals(attribs["KEY_b"], monitor_b.attribute_value)
def __create_test_instance(self, use_pipelining=False):
tmp_dir = tempfile.mkdtemp()
config_dir = os.path.join(tmp_dir, "config")
data_dir = os.path.join(tmp_dir, "data")
log_dir = os.path.join(tmp_dir, "log")
os.mkdir(data_dir)
os.mkdir(config_dir)
os.mkdir(log_dir)
self.__test_log_file = os.path.join(tmp_dir, "test.log")
fp = open(self.__test_log_file, "w")
fp.close()
config_file = os.path.join(config_dir, "agentConfig.json")
config_fragments_dir = os.path.join(config_dir, "configs.d")
os.makedirs(config_fragments_dir)
logs_json_array = JsonArray()
logs_json_array.add(JsonObject(path=self.__test_log_file))
pipeline_threshold = 1.1
if use_pipelining:
pipeline_threshold = 0.0
fp = open(config_file, "w")
fp.write(
json_lib.serialize(
JsonObject(
api_key="fake",
logs=logs_json_array,
pipeline_threshold=pipeline_threshold,
)))
fp.close()
default_paths = DefaultPaths(log_dir, config_file, data_dir)
config = Configuration(config_file, default_paths, None)
config.parse()
# noinspection PyTypeChecker
self._controller = TestableCopyingManager(config, []).controller
return self._controller
def __parse_array(self):
"""Parse a JSON array. The scanner must be at the first '['."""
array_start = self.__scanner.position
self.__scanner.read_ubyte()
array = JsonArray()
while True:
# Check for end-of-array.
if self.__peek_next_non_whitespace() == ']':
self.__scanner.read_ubyte()
return array
self.__peek_next_non_whitespace() # skip any whitespace
# TODO: If we ever want to put in annotated supported, uncomment the pos lines.
# value_start_pos = self.__scanner.position
array.add(self.parse_value())
# value_end_pos = self.__scanner.position
# value_comma_pos = -1
c = self.__peek_next_non_whitespace()
if c is None:
self.__error("Array has no terminating '['", array_start)
elif c == ']':
# do nothing we'll process the ']' back around at the top of
# the loop.
continue
elif c == ',':
self.__scanner.read_ubyte()
# value_comma_pos = self.__scanner.position
else:
if self.__preceding_line_break() and self.allow_missing_commas:
# proceed, inferring a comma
continue
else:
self.__error(
"Unexpected character [%s] in array... are you "
"missing a comma?" % c)
def test_jsonobject_nested_jsonarray2(self):
self.__test_encode_decode(
r'{"a":[1,2,3,[1,2,3]]}',
JsonObject({u"a": JsonArray(1, 2, 3, JsonArray(1, 2, 3))}),
)
def test_set_item(self):
x = JsonArray('bye', 3)
x[0] = 'hi'
self.assertEquals(x[0], 'hi')
self.assertRaises(IndexError, x.__setitem__, 5, 'foo')
def test_jsonobject_nested_jsonarray(self):
self.__test_encode_decode(r'{"a":[1,2,3]}', JsonObject({u'a': JsonArray(1, 2, 3)}))
def test_set_item(self):
x = JsonArray("bye", 3)
x[0] = "hi"
self.assertEquals(x[0], "hi")
self.assertRaises(IndexError, x.__setitem__, 5, "foo")
def test_constructor(self):
x = JsonArray("hi", True)
self.assertEquals(len(x), 2)
self.assertEquals(x[0], "hi")
self.assertEquals(x[1], True)
def test_basic_case(self):
self.__write_file_with_separator_conversion(""" {
api_key: "hi there",
logs: [ { path:"/var/log/tomcat6/access.log"} ]
}
""")
config = self.__create_test_configuration_instance()
config.parse()
self.assertEquals(config.api_key, "hi there")
self.assertPathEquals(config.agent_log_path, '/var/log/scalyr-agent-2')
self.assertPathEquals(config.agent_data_path,
'/var/lib/scalyr-agent-2')
self.assertEquals(config.additional_monitor_module_paths, '')
self.assertEquals(config.config_directory, self.__config_fragments_dir)
self.assertEquals(config.implicit_metric_monitor, True)
self.assertEquals(config.implicit_agent_log_collection, True)
self.assertFalse(config.use_unsafe_debugging)
self.assertEquals(config.scalyr_server, 'https://agent.scalyr.com')
self.assertEquals(len(config.server_attributes), 1)
self.assertTrue('serverHost' in config.server_attributes)
self.assertEquals(config.global_monitor_sample_interval, 30.0)
self.assertEquals(config.max_allowed_request_size, 1 * 1024 * 1024)
self.assertEquals(config.min_allowed_request_size, 100 * 1024)
self.assertEquals(config.min_request_spacing_interval, 1.0)
self.assertEquals(config.max_request_spacing_interval, 5.0)
self.assertEquals(config.high_water_bytes_sent, 100 * 1024)
self.assertEquals(config.high_water_request_spacing_adjustment, 0.6)
self.assertEquals(config.low_water_bytes_sent, 20 * 1024)
self.assertEquals(config.low_water_request_spacing_adjustment, 1.5)
self.assertEquals(config.failure_request_spacing_adjustment, 1.5)
self.assertEquals(config.request_too_large_adjustment, 0.5)
self.assertEquals(config.debug_level, 0)
self.assertEquals(config.request_deadline, 60.0)
self.assertEquals(config.max_line_size, 3400)
self.assertEquals(config.max_log_offset_size, 5 * 1024 * 1024)
self.assertEquals(config.max_existing_log_offset_size,
100 * 1024 * 1024)
self.assertEquals(config.max_sequence_number, 1024**4)
self.assertEquals(config.line_completion_wait_time, 5 * 60)
self.assertEquals(config.read_page_size, 64 * 1024)
self.assertEquals(config.copy_staleness_threshold, 15 * 60)
self.assertEquals(config.log_deletion_delay, 10 * 60)
self.assertTrue(config.ca_cert_path.endswith('ca_certs.crt'))
self.assertTrue(config.verify_server_certificate)
self.assertFalse(config.debug_init)
self.assertFalse(config.pidfile_advanced_reuse_guard)
self.assertEquals(len(config.log_configs), 2)
self.assertPathEquals(config.log_configs[0].get_string('path'),
'/var/log/tomcat6/access.log')
self.assertEquals(config.log_configs[0].get_json_object('attributes'),
JsonObject())
self.assertEquals(
config.log_configs[0].get_json_array('sampling_rules'),
JsonArray())
self.assertEquals(
config.log_configs[0].get_json_array('redaction_rules'),
JsonArray())
self.assertPathEquals(config.log_configs[1].get_string('path'),
'/var/log/scalyr-agent-2/agent.log')
self.assertEquals(len(config.monitor_configs), 0)
def _process_annotation_items(items):
""" Process annotation items after the scalyr config prefix has been stripped
"""
result = {}
def sort_annotation(pair):
(key, value) = pair
m = SCALYR_ANNOTATION_ELEMENT_RE.match(key)
if m:
root_key = m.group(1)
if _is_int(root_key):
return int(root_key)
return root_key
return key
def sort_numeric(pair):
(key, value) = pair
if _is_int(key):
return int(key)
return key
# sort dict by the value of the first sub key (up to the first '.')
# this ensures that all items of the same key are processed together
sorted_items = sorted(items.iteritems(), key=sort_annotation)
current_object = None
previous_key = None
is_array = False
is_object = False
for (key, value) in sorted_items:
# split out the sub key from the rest of the key
m = SCALYR_ANNOTATION_ELEMENT_RE.match(key)
if m:
root_key = m.group(1)
child_key = m.group(2)
# check for mixed list and dict keys and raise an error if they exist
if _is_int(root_key):
is_array = True
else:
is_object = True
if is_object == is_array:
raise BadAnnotationConfig(
"Annotation cannot be both a dict and a list for '%s'. Current key: %s, previous key: %s"
% (key, str(root_key), str(previous_key)))
# create an empty object if None exists
if current_object is None:
current_object = {}
# else if the keys are different which means we have a new key,
# so add the current object to the list of results and create a new object
elif previous_key is not None and root_key != previous_key:
result[previous_key] = _process_annotation_items(
current_object)
current_object = {}
current_object[child_key] = value
previous_key = root_key
else: # no more subkeys so just process as the full key
# check for mixed list and dict keys and raise an error if they exist
if _is_int(key):
is_array = True
else:
is_object = True
if is_object == is_array:
raise BadAnnotationConfig(
"Annotation cannot be both a dict and a list. Current key: %s, previous key: %s"
% (key, str(previous_key)))
# if there was a previous key
if previous_key is not None and current_object is not None:
# stick it in the result
result[previous_key] = _process_annotation_items(
current_object)
# add the current value to the result
result[key] = value
current_object = None
previous_key = None
# add the final object if there was one
if previous_key is not None and current_object is not None:
result[previous_key] = _process_annotation_items(current_object)
# if the result should be an array, return values as a JsonArray, sorted by numeric order of keys
if is_array:
result = JsonArray(
*[r[1] for r in sorted(result.iteritems(), key=sort_numeric)])
else:
# return values as a JsonObject
result = JsonObject(content=result)
return result
def _process_annotation_items(items, hyphens_as_underscores):
""" Process annotation items after the scalyr config prefix has been stripped
"""
def sort_annotation(pair):
(key, value) = pair
m = SCALYR_ANNOTATION_ELEMENT_RE.match(key)
if m:
root_key = m.group(1)
if _is_int(root_key):
return int(root_key)
return root_key
return key
def sort_numeric(pair):
(key, value) = pair
if _is_int(key):
return int(key)
return key
def normalize_key_name(key, convert_hyphens):
"""
Normalizes the name of the key by converting any hyphens to underscores (or not)
depending on the `convert_hyphens` parameter.
Typically, `convert_hypens` will be false when converting k8s annotations because
label and annotation keys in k8s can contain underscores.
Keys for docker labels however cannot container underscores, therefore `convert_hyphens`
can be used to convert any label keys to use underscores, which are expected by various
log_config options.
This function means the code that processes the labels/annotations doesn't need to care
if it is running under docker or k8s, because the root caller decides whether or not hyphens
need converting.
@param key: string - a key for an annotation/label
@param convert_hyphens: bool - if True, any hyphens in the `key` parameter will be
converted to underscores
"""
if convert_hyphens:
key = key.replace("-", "_")
return key
result = {}
# sort dict by the value of the first sub key (up to the first '.')
# this ensures that all items of the same key are processed together
sorted_items = sorted(six.iteritems(items), key=sort_annotation)
current_object = None
previous_key = None
is_array = False
is_object = False
for (key, value) in sorted_items:
# split out the sub key from the rest of the key
m = SCALYR_ANNOTATION_ELEMENT_RE.match(key)
if m:
root_key = m.group(1)
child_key = m.group(2)
# check for mixed list and dict keys and raise an error if they exist
if _is_int(root_key):
is_array = True
else:
is_object = True
if is_object == is_array:
raise BadAnnotationConfig(
"Annotation cannot be both a dict and a list for '%s'. Current key: %s, previous key: %s"
% (key, str(root_key), str(previous_key)))
# create an empty object if None exists
if current_object is None:
current_object = {}
# else if the keys are different which means we have a new key,
# so add the current object to the list of results and create a new object
elif previous_key is not None and root_key != previous_key:
updated_key = normalize_key_name(previous_key,
hyphens_as_underscores)
result[updated_key] = _process_annotation_items(
current_object, hyphens_as_underscores)
current_object = {}
current_object[child_key] = value
previous_key = root_key
else: # no more subkeys so just process as the full key
# check for mixed list and dict keys and raise an error if they exist
if _is_int(key):
is_array = True
else:
is_object = True
if is_object == is_array:
raise BadAnnotationConfig(
"Annotation cannot be both a dict and a list. Current key: %s, previous key: %s"
% (key, str(previous_key)))
# if there was a previous key
if previous_key is not None and current_object is not None:
# stick it in the result
updated_key = normalize_key_name(previous_key,
hyphens_as_underscores)
result[updated_key] = _process_annotation_items(
current_object, hyphens_as_underscores)
# add the current value to the result
updated_key = normalize_key_name(key, hyphens_as_underscores)
result[updated_key] = value
current_object = None
previous_key = None
# add the final object if there was one
if previous_key is not None and current_object is not None:
updated_key = normalize_key_name(previous_key, hyphens_as_underscores)
result[updated_key] = _process_annotation_items(
current_object, hyphens_as_underscores)
# if the result should be an array, return values as a JsonArray, sorted by numeric order of keys
if is_array:
result = JsonArray(
*[r[1] for r in sorted(six.iteritems(result), key=sort_numeric)])
else:
# return values as a JsonObject
result = JsonObject(content=result)
return result
def test_jsonarray(self):
self.__test_encode_decode(r"[1,2,3]", JsonArray(1, 2, 3))