def __init__(self, configuration, monitors):
CopyingManager.__init__(self, configuration, monitors)
# Approach: We will override key methods of CopyingManager, blocking them from returning until the controller
# tells it to proceed. This allows us to then do things like write new log lines while the CopyingManager is
# blocked. Coordinating the communication between the two threads is done using two condition variables.
# We changed the CopyingManager to block in three places: while it is sleeping before it starts a new loop,
# when it invokes ``_send_events`` to send a new request, and when it blocks to receive the response.
# These three states or referred to as "sleeping", "blocked_on_send", "blocked_on_receive".
#
# This cv protects all of the variables written by the CopyingManager thread.
self.__test_state_cv = threading.Condition()
# Which state the CopyingManager is currently blocked in -- "sleeping", "blocked_on_send", "blocked_on_receive"
self.__test_state = None
# The number of times the CopyingManager has blocked.
self.__test_state_changes = 0
# Whether or not the CopyingManager should stop.
self.__test_stopping = False
# Written by CopyingManager. The last AddEventsRequest request passed into ``_send_events``.
self.__captured_request = None
# Protected by __test_state_cv. The status message to return for the next call to ``_send_events``.
self.__pending_response = None
# This cv protects __advance_requests and is used mainly by the testing thread.
self.__advance_requests_cv = threading.Condition()
# This is incremented everytime the controller wants the CopyingManager to advance to the next blocking state,
# regardless of which state it is in.
self.__advance_requests = 0
self.__controller = TestableCopyingManager.TestController(self)
def __init__(self, configuration, monitors):
CopyingManager.__init__(self, configuration, monitors)
# Approach: We will override key methods of CopyingManager, blocking them from returning until the controller
# tells it to proceed. This allows us to then do things like write new log lines while the CopyingManager is
# blocked. Coordinating the communication between the two threads is done using one condition variable.
# We changed the CopyingManager to block in three places: while it is sleeping before it starts a new loop,
# when it invokes `_send_events` to send a new request, and when it blocks to receive the response.
# These three states are referred to as 'sleeping', 'sending', 'responding'.
#
# The CopyingManager will have state to record where it should next block (i.e., if it should block at
# 'sleeping' when it attempts to sleep). The test controller will manipulate this state, notifying changes on
# the condition variable. The CopyingManager will block in this state (and indicate it is blocked) until the
# test controller sets a new state to block at.
#
# This cv protects all of the variables written by the CopyingManager thread.
self.__test_state_cv = threading.Condition()
# Which state the CopyingManager should block in -- "sleeping", "sending", "responding"
# We initialize it to a special value "all" so that it stops as soon the CopyingManager starts up.
self.__test_stop_state = 'all'
# If not none, a state the test must pass through before it tries to stop at `__test_stop_state`.
# If this transition is not observed by the time it does get to the stop state, an assertion is thrown.
self.__test_required_transition = None
# Whether or not the CopyingManager is stopped at __test_stop_state.
self.__test_is_stopped = False
# Written by CopyingManager. The last AddEventsRequest request passed into ``_send_events``.
self.__captured_request = None
# Protected by __test_state_cv. The status message to return for the next call to ``_send_events``.
self.__pending_response = None
self.__controller = TestableCopyingManager.TestController(self)
def __init__(self, configuration, monitors):
CopyingManager.__init__(self, configuration, monitors)
# Approach: We will override key methods of CopyingManager, blocking them from returning until the controller
# tells it to proceed. This allows us to then do things like write new log lines while the CopyingManager is
# blocked. Coordinating the communication between the two threads is done using two condition variables.
# We changed the CopyingManager to block in three places: while it is sleeping before it starts a new loop,
# when it invokes ``_send_events`` to send a new request, and when it blocks to receive the response.
# These three states or referred to as "sleeping", "blocked_on_send", "blocked_on_receive".
#
# This cv protects all of the variables written by the CopyingManager thread.
self.__test_state_cv = threading.Condition()
# Which state the CopyingManager is currently blocked in -- "sleeping", "blocked_on_send", "blocked_on_receive"
self.__test_state = None
# The number of times the CopyingManager has blocked.
self.__test_state_changes = 0
# Whether or not the CopyingManager should stop.
self.__test_stopping = False
# Written by CopyingManager. The last AddEventsRequest request passed into ``_send_events``.
self.__captured_request = None
# Protected by __test_state_cv. The status message to return for the next call to ``_send_events``.
self.__pending_response = None
# This cv protects __advance_requests and is used mainly by the testing thread.
self.__advance_requests_cv = threading.Condition()
# This is incremented everytime the controller wants the CopyingManager to advance to the next blocking state,
# regardless of which state it is in.
self.__advance_requests = 0
self.__controller = TestableCopyingManager.TestController(self)
def __create_test_instance(self, configuration_logs_entry,
monitors_log_configs):
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
logs_json_array = JsonArray()
for entry in configuration_logs_entry:
logs_json_array.add(JsonObject(content=entry))
fp = open(config_file, 'w')
fp.write(
json_lib.serialize(JsonObject(api_key='fake',
logs=logs_json_array)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths)
config.parse()
self.__monitor_fake_instances = []
for monitor_log_config in monitors_log_configs:
self.__monitor_fake_instances.append(
FakeMonitor(monitor_log_config))
# noinspection PyTypeChecker
return CopyingManager(config, self.__monitor_fake_instances)
def test_get_server_attribute_no_override(self):
logs_json_array = JsonArray()
config = ScalyrTestUtils.create_configuration(
extra_toplevel_config={'logs': logs_json_array})
self.__monitor_fake_instances = []
monitor_a = FakeMonitor1({'path': 'testA.log'},
id='a',
attribute_key='common_key')
monitor_b = FakeMonitor1({'path': 'testB.log'},
id='b',
attribute_key='common_key')
self.__monitor_fake_instances.append(monitor_a)
self.__monitor_fake_instances.append(monitor_b)
copy_manager = CopyingManager(config, self.__monitor_fake_instances)
if monitor_a.access_order < monitor_b.access_order:
first_accessed = monitor_a
second_accessed = monitor_b
else:
first_accessed = monitor_b
second_accessed = monitor_a
self.assertLess(first_accessed.access_order,
second_accessed.access_order)
self.assertEquals(
copy_manager.expanded_server_attributes['common_key'],
first_accessed.attribute_value)
def stop_manager(self, wait_on_join=True, join_timeout=5):
"""Stops the manager's thread.
@param wait_on_join: Whether or not to wait on thread to finish.
@param join_timeout: The number of seconds to wait on the join.
@type wait_on_join: bool
@type join_timeout: float
@return:
@rtype:
"""
# We need to do some extra work here in case the CopyingManager thread is currently in a blocked state.
# We need to tell it to keep running.
self.__test_state_cv.acquire()
self.__test_stop_state = None
self.__test_state_cv.notifyAll()
self.__test_state_cv.release()
CopyingManager.stop_manager(self,
wait_on_join=wait_on_join,
join_timeout=join_timeout)
def stop_manager(self, wait_on_join=True, join_timeout=5):
"""Stops the manager's thread.
@param wait_on_join: Whether or not to wait on thread to finish.
@param join_timeout: The number of seconds to wait on the join.
@type wait_on_join: bool
@type join_timeout: float
@return:
@rtype:
"""
# We need to do some extra work here in case the CopyingManager thread is currently in a blocked state.
# We need to tell it to advance.
self.__test_state_cv.acquire()
self.__test_stopping = True
self.__test_state_cv.release()
self.__advance_requests_cv.acquire()
self.__advance_requests += 1
self.__advance_requests_cv.notifyAll()
self.__advance_requests_cv.release()
CopyingManager.stop_manager(self, wait_on_join=wait_on_join, join_timeout=join_timeout)
def __create_test_instance(self, configuration_logs_entry, monitors_log_configs):
logs_json_array = JsonArray()
for entry in configuration_logs_entry:
logs_json_array.add(JsonObject(content=entry))
config = ScalyrTestUtils.create_configuration(extra_toplevel_config={'logs': logs_json_array})
self.__monitor_fake_instances = []
for monitor_log_config in monitors_log_configs:
self.__monitor_fake_instances.append(FakeMonitor(monitor_log_config))
# noinspection PyTypeChecker
return CopyingManager(config, self.__monitor_fake_instances)
def run_test():
manager_poll_interval = 30
fake_clock = FakeClock()
monitors_manager, config = ScalyrTestUtils.create_test_monitors_manager(
config_monitors=[{
'module':
"scalyr_agent.builtin_monitors.kubernetes_monitor",
}],
extra_toplevel_config={
'user_agent_refresh_interval': manager_poll_interval
},
null_logger=True,
fake_clock=fake_clock,
)
copying_manager = CopyingManager(config, monitors_manager.monitors)
def test_get_server_attribute(self):
logs_json_array = JsonArray()
config = ScalyrTestUtils.create_configuration(
extra_toplevel_config={'logs': logs_json_array})
self.__monitor_fake_instances = []
monitor_a = FakeMonitor1({'path': 'testA.log'},
id='a',
attribute_key='KEY_a')
monitor_b = FakeMonitor1({'path': 'testB.log'},
id='b',
attribute_key='KEY_b')
self.__monitor_fake_instances.append(monitor_a)
self.__monitor_fake_instances.append(monitor_b)
copy_manager = CopyingManager(config, self.__monitor_fake_instances)
attribs = copy_manager.expanded_server_attributes
self.assertEquals(attribs['KEY_a'], monitor_a.attribute_value)
self.assertEquals(attribs['KEY_b'], monitor_b.attribute_value)
def test_get_server_attribute(self):
logs_json_array = JsonArray()
config = ScalyrTestUtils.create_configuration(
extra_toplevel_config={"logs": logs_json_array})
self.__monitor_fake_instances = []
monitor_a = FakeMonitor1({"path": "testA.log"},
id="a",
attribute_key="KEY_a")
monitor_b = FakeMonitor1({"path": "testB.log"},
id="b",
attribute_key="KEY_b")
self.__monitor_fake_instances.append(monitor_a)
self.__monitor_fake_instances.append(monitor_b)
copy_manager = CopyingManager(config, self.__monitor_fake_instances)
attribs = copy_manager.expanded_server_attributes
self.assertEquals(attribs["KEY_a"], monitor_a.attribute_value)
self.assertEquals(attribs["KEY_b"], monitor_b.attribute_value)
def run_test():
manager_poll_interval = 30
fake_clock = FakeClock()
monitors_manager, config = ScalyrTestUtils.create_test_monitors_manager(
config_monitors=[{
"module":
"scalyr_agent.builtin_monitors.kubernetes_monitor"
}],
extra_toplevel_config={
"user_agent_refresh_interval": manager_poll_interval
},
null_logger=True,
fake_clock=fake_clock,
)
copying_manager = CopyingManager(config, monitors_manager.monitors)
self.assertEquals(
copying_manager._CopyingManager__expanded_server_attributes.
get("_k8s_ver"),
"star",
)
def __init__(self, configuration, monitors):
CopyingManager.__init__(self, configuration, monitors)
TestableCopyingManagerFlowController.__init__(self, configuration)
# do this just to tell static analyzer that this is a testable instances.
self._log_matchers = self._log_matchers # type: List[TestableLogMatcher]
def test_environment_aware_module_params(self, mock_docker):
# Define test values here for all k8s and k8s_event monitor config params that are environment aware.
# Be sure to use non-default test values
TEST_INT = 123456789
TEST_STRING = "dummy string"
TEST_ARRAY_OF_STRINGS = ["s1", "s2", "s3"]
STANDARD_PREFIX = "_STANDARD_PREFIX_" # env var is SCALYR_<param_name>
# The following map contains config params to be tested
# config_param_name: (custom_env_name, test_value)
docker_testmap = {
"container_check_interval": (STANDARD_PREFIX, TEST_INT, int),
"docker_api_version": (STANDARD_PREFIX, TEST_STRING, str),
"docker_log_prefix": (STANDARD_PREFIX, TEST_STRING, str),
"log_mode": ("SCALYR_DOCKER_LOG_MODE", TEST_STRING, str),
"docker_raw_logs": (
STANDARD_PREFIX,
False,
bool,
), # test config file is set to True
"docker_percpu_metrics": (
STANDARD_PREFIX,
True,
bool,
), # test config file is set to False
"metrics_only": ("SCALYR_DOCKER_METRICS_ONLY", True, bool),
"container_globs":
(STANDARD_PREFIX, TEST_ARRAY_OF_STRINGS, ArrayOfStrings),
"container_globs_exclude": (
STANDARD_PREFIX,
TEST_ARRAY_OF_STRINGS,
ArrayOfStrings,
),
"report_container_metrics": (STANDARD_PREFIX, False, bool),
"label_include_globs": (
STANDARD_PREFIX,
TEST_ARRAY_OF_STRINGS,
ArrayOfStrings,
),
"label_exclude_globs": (
STANDARD_PREFIX,
TEST_ARRAY_OF_STRINGS,
ArrayOfStrings,
),
"labels_as_attributes": (STANDARD_PREFIX, True, bool),
"label_prefix": (STANDARD_PREFIX, TEST_STRING, str),
"use_labels_for_log_config": (STANDARD_PREFIX, False, bool),
}
# Fake the environment varaibles
for key, value in docker_testmap.items():
custom_name = value[0]
env_name = (("SCALYR_%s" % key).upper() if custom_name
== STANDARD_PREFIX else custom_name.upper())
envar_value = str(value[1])
if value[2] == ArrayOfStrings:
# Array of strings should be entered into environment in the user-preferred format
# which is without square brackets and quotes around each element
envar_value = envar_value[1:-1] # strip square brackets
envar_value = envar_value.replace("'", "")
else:
envar_value = (envar_value.lower()
) # lower() needed for proper bool encoding
os.environ[env_name] = envar_value
self._write_file_with_separator_conversion(""" {
logs: [ { path:"/var/log/tomcat6/$DIR_VAR.log" }],
api_key: "abcd1234",
}
""")
self._write_config_fragment_file_with_separator_conversion(
"docker.json",
""" {
"monitors": [
{
module: "scalyr_agent.builtin_monitors.docker_monitor",
docker_raw_logs: true
}
]
}
""",
)
config = self._create_test_configuration_instance()
config.parse()
monitors_manager, mock_logger = self._make_monitors_manager(config)
docker_monitor = monitors_manager.monitors[0]
# All environment-aware params defined in the docker monitor must be gested
self.assertEquals(
set(docker_testmap.keys()),
set(docker_monitor._config._environment_aware_map.keys()),
)
# Verify module-level conflicts between env var and config file are logged at module-creation time
mock_logger.warn.assert_called_with(
"Conflicting values detected between scalyr_agent.builtin_monitors.docker_monitor config file "
"parameter `docker_raw_logs` and the environment variable `SCALYR_DOCKER_RAW_LOGS`. "
"Ignoring environment variable.",
limit_once_per_x_secs=300,
limit_key=
"config_conflict_scalyr_agent.builtin_monitors.docker_monitor_docker_raw_logs_SCALYR_DOCKER_RAW_LOGS",
)
CopyingManager(config, monitors_manager.monitors)
# Override Agent Logger to prevent writing to disk
for monitor in monitors_manager.monitors:
monitor._logger = FakeAgentLogger("fake_agent_logger")
# Verify environment variable values propagate into DockerMonitor monitor MonitorConfig
monitor_2_testmap = {
docker_monitor: docker_testmap,
}
for monitor, testmap in monitor_2_testmap.items():
for key, value in testmap.items():
test_val, convert_to = value[1:]
if key in ["docker_raw_logs"]:
# Keys were defined in config files so should not have changed
self.assertNotEquals(
test_val,
monitor._config.get(key, convert_to=convert_to))
else:
# Keys were empty in config files so they take on environment values
materialized_value = monitor._config.get(
key, convert_to=convert_to)
if hasattr(test_val, "__iter__"):
self.assertEquals([x1 for x1 in test_val],
[x2 for x2 in materialized_value])
else:
self.assertEquals(test_val, materialized_value)
def test_environment_aware_module_params(self, mock_docker):
# Define test values here for all k8s and k8s_event monitor config params that are environment aware.
# Be sure to use non-default test values
TEST_INT = 123456789
TEST_FLOAT = 1234567.89
TEST_STRING = "dummy string"
TEST_PARSE_FORMAT = "cri"
TEST_ARRAY_OF_STRINGS = ["s1", "s2", "s3"]
STANDARD_PREFIX = "_STANDARD_PREFIX_" # env var is SCALYR_<param_name>
# The following map contains config params to be tested
# config_param_name: (custom_env_name, test_value)
k8s_testmap = {
"container_check_interval": (STANDARD_PREFIX, TEST_INT, int),
"initial_stopped_container_collection_window": (
STANDARD_PREFIX,
TEST_INT,
int,
),
"docker_max_parallel_stats": (STANDARD_PREFIX, TEST_INT, int),
"docker_percpu_metrics": (STANDARD_PREFIX, True, bool),
"container_globs":
(STANDARD_PREFIX, TEST_ARRAY_OF_STRINGS, ArrayOfStrings),
"report_container_metrics": (STANDARD_PREFIX, False, bool),
"report_k8s_metrics": (STANDARD_PREFIX, True, bool),
"k8s_ignore_pod_sandboxes": (STANDARD_PREFIX, False, bool),
"k8s_include_all_containers": (STANDARD_PREFIX, False, bool),
"k8s_sidecar_mode": (STANDARD_PREFIX, True, bool),
"k8s_parse_format":
(STANDARD_PREFIX, TEST_PARSE_FORMAT, six.text_type),
"k8s_always_use_cri": (STANDARD_PREFIX, True, bool),
"k8s_cri_query_filesystem": (STANDARD_PREFIX, True, bool),
"k8s_always_use_docker": (STANDARD_PREFIX, True, bool),
"k8s_kubelet_host_ip": (STANDARD_PREFIX, False, bool),
"k8s_kubelet_api_url_template": (STANDARD_PREFIX, False, bool),
"gather_k8s_pod_info": (STANDARD_PREFIX, True, bool),
}
k8s_events_testmap = {
"max_log_size": ("SCALYR_K8S_MAX_LOG_SIZE", TEST_INT, int),
"max_log_rotations":
("SCALYR_K8S_MAX_LOG_ROTATIONS", TEST_INT, int),
"log_flush_delay":
("SCALYR_K8S_LOG_FLUSH_DELAY", TEST_FLOAT, float),
"message_log":
("SCALYR_K8S_MESSAGE_LOG", TEST_STRING, six.text_type),
"event_object_filter": (
"SCALYR_K8S_EVENT_OBJECT_FILTER",
TEST_ARRAY_OF_STRINGS,
ArrayOfStrings,
),
"leader_check_interval": (
"SCALYR_K8S_LEADER_CHECK_INTERVAL",
TEST_INT,
int,
),
"leader_node":
("SCALYR_K8S_LEADER_NODE", TEST_STRING, six.text_type),
"check_labels": ("SCALYR_K8S_CHECK_LABELS", True, bool),
"ignore_master": ("SCALYR_K8S_IGNORE_MASTER", False, bool),
}
# Fake the environment variables
for map in [k8s_testmap, k8s_events_testmap]:
for key, value in map.items():
custom_name = value[0]
env_name = (("SCALYR_%s" % key).upper() if custom_name
== STANDARD_PREFIX else custom_name.upper())
param_value = value[1]
if value[2] == ArrayOfStrings:
# Array of strings should be entered into environment in the user-preferred format
# which is without square brackets and quotes around each element
envar_value = "[{0}]".format(", ".join(param_value))
else:
envar_value = six.text_type(param_value)
envar_value = (envar_value.lower()
) # lower() needed for proper bool encoding
os.environ[env_name] = envar_value
self._write_file_with_separator_conversion(""" {
logs: [ { path:"/var/log/tomcat6/$DIR_VAR.log" }],
api_key: "abcd1234",
}
""")
self._write_config_fragment_file_with_separator_conversion(
"k8s.json",
""" {
"monitors": [
{
"module": "scalyr_agent.builtin_monitors.kubernetes_monitor",
"report_k8s_metrics": false,
},
{
"module": "scalyr_agent.builtin_monitors.kubernetes_events_monitor"
}
]
}
""",
)
monitors_manager, config, mock_logger = self._create_test_objects()
k8s_monitor = monitors_manager.monitors[0]
k8s_events_monitor = monitors_manager.monitors[1]
# All environment-aware params defined in the k8s and k8s_events monitors must be tested
self.assertEquals(
set(k8s_testmap.keys()),
set(k8s_monitor._config._environment_aware_map.keys()),
)
self.assertEquals(
set(k8s_events_testmap.keys()),
set(k8s_events_monitor._config._environment_aware_map.keys()),
)
# Verify module-level conflicts between env var and config file are logged at module-creation time
mock_logger.warn.assert_called_with(
"Conflicting values detected between scalyr_agent.builtin_monitors.kubernetes_monitor config file "
"parameter `report_k8s_metrics` and the environment variable `SCALYR_REPORT_K8S_METRICS`. "
"Ignoring environment variable.",
limit_once_per_x_secs=300,
limit_key=
"config_conflict_scalyr_agent.builtin_monitors.kubernetes_monitor_report_k8s_metrics_SCALYR_REPORT_K8S_METRICS",
)
CopyingManager(config, monitors_manager.monitors)
# Override Agent Logger to prevent writing to disk
for monitor in monitors_manager.monitors:
monitor._logger = FakeAgentLogger("fake_agent_logger")
# Verify environment variable values propagate into kubernetes monitor MonitorConfig
monitor_2_testmap = {
k8s_monitor: k8s_testmap,
k8s_events_monitor: k8s_events_testmap,
}
for monitor, testmap in monitor_2_testmap.items():
for key, value in testmap.items():
test_val, convert_to = value[1:]
if key in ["report_k8s_metrics", "api_key"]:
# Keys were defined in config files so should not have changed
self.assertNotEquals(
test_val,
monitor._config.get(key, convert_to=convert_to))
else:
# Keys were empty in config files so they take on environment values
materialized_value = monitor._config.get(
key, convert_to=convert_to)
if hasattr(test_val, "__iter__"):
self.assertEquals([x1 for x1 in test_val],
[x2 for x2 in materialized_value])
else:
self.assertEquals(test_val, materialized_value)
