def _test_standalone_smoke(agent_installation_type, python_version=None):
"""
Agent standalone test to run within the same machine.
"""
if agent_installation_type == DEV_INSTALL:
os.chdir(get_package_root())
print("Agent host name: {0}".format(
compat.os_environ_unicode["AGENT_HOST_NAME"]))
runner = AgentRunner(agent_installation_type, enable_debug_log=True)
if python_version:
runner.switch_version(python_version)
agent_log_verifier = AgentLogVerifier(
runner, compat.os_environ_unicode["SCALYR_SERVER"])
data_json_verifier = DataJsonVerifier(
runner, compat.os_environ_unicode["SCALYR_SERVER"])
system_metrics_verifier = SystemMetricsVerifier(
runner, compat.os_environ_unicode["SCALYR_SERVER"])
process_metrics_verifier = ProcessMetricsVerifier(
runner, compat.os_environ_unicode["SCALYR_SERVER"])
# NOTE: It's important that the file exists before starting the agent otherwise it won't be
# consumed by the agent and the tests will fail.
agent_logs_dir_path = six.text_type(runner.agent_logs_dir_path)
data_log_path = six.text_type(data_json_verifier._data_json_log_path)
if os.path.exists(data_log_path):
os.utime(data_log_path, None)
else:
try:
os.makedirs(agent_logs_dir_path)
except OSError:
pass
with open(data_log_path, "a") as _:
pass
runner.start()
time.sleep(1)
print("Verify 'agent.log'")
assert agent_log_verifier.verify(
), "Verification of the file: 'agent.log' failed"
print("Verify 'linux_system_metrics.log'")
assert (system_metrics_verifier.verify()
), "Verification of the file: 'linux_system_metrics.log' failed"
print("Verify 'linux_process_metrics.log'")
assert (process_metrics_verifier.verify()
), "Verification of the file: 'linux_process_metrics.log' failed"
assert data_json_verifier.verify(
), "Verification of the file: 'data.json' failed"
runner.stop()
def test_centos_test_versions(request):
runner = AgentRunner(PACKAGE_INSTALL)
common_version_test(
runner,
install_rpm,
remove_rpm,
None,
"2.5.1",
"2.5.1",
"3.4.1",
install_fails=True,
)
# We only test that scenario under CentOS 8 where "python" binary is not available
common_version_test(
runner,
install_rpm,
remove_rpm,
"config_main.py",
"",
"2.5.1",
"3.4.1",
install_fails=True,
)
common_version_test(runner, install_rpm, remove_rpm, "config_main_py2.py",
"2.5.1", "", "3.4.1")
common_version_test(runner, install_rpm, remove_rpm, "config_main_py3.py",
"2.5.1", "2.5.1", "")
common_version_test(runner, install_rpm, remove_rpm, "config_main_py2.py",
"", "", "")
common_version_test(runner, install_rpm, remove_rpm, "config_main_py2.py",
"2.5.1", "", "")
common_version_test(runner, install_rpm, remove_rpm, "config_main_py3.py",
"", "2.5.1", "")
def common_test_ubuntu_versions():
runner = AgentRunner(PACKAGE_INSTALL)
common_version_test(
runner,
install_deb,
remove_deb,
None,
"2.5.1",
"2.5.1",
"3.4.1",
install_fails=True,
)
common_version_test(
runner, install_deb, remove_deb, "config_main.py", "", "2.5.1", "3.4.1"
)
common_version_test(
runner, install_deb, remove_deb, "config_main_py2.py", "2.5.1", "", "3.4.1"
)
common_version_test(
runner, install_deb, remove_deb, "config_main_py3.py", "2.5.1", "2.5.1", ""
)
common_version_test(runner, install_deb, remove_deb, "config_main.py", "", "", "")
common_version_test(
runner, install_deb, remove_deb, "config_main_py2.py", "2.5.1", "", ""
)
common_version_test(
runner, install_deb, remove_deb, "config_main.py", "", "2.5.1", ""
)
def test_centos_test_versions(request):
runner = AgentRunner(PACKAGE_INSTALL)
common_version_test(
runner,
install_rpm,
remove_rpm,
None,
"2.5.1",
"2.5.1",
"3.4.1",
install_fails=True,
)
common_version_test(
runner, install_rpm, remove_rpm, "config_main_py2.py", "2.5.1", "", "3.4.1"
)
common_version_test(
runner, install_rpm, remove_rpm, "config_main_py3.py", "2.5.1", "2.5.1", ""
)
common_version_test(runner, install_rpm, remove_rpm, "config_main.py", "", "", "")
common_version_test(
runner, install_rpm, remove_rpm, "config_main_py2.py", "2.5.1", "", ""
)
common_version_test(
runner, install_rpm, remove_rpm, "config_main.py", "", "2.5.1", ""
)
def test_standalone_agent_kill():
runner = AgentRunner(
DEV_INSTALL,
enable_debug_log=True,
workers_type="process",
workers_session_count=2,
)
runner.start()
logging.info("Agent started.")
time.sleep(1)
process, worker_pids, children, workers_processes = _perform_workers_check(runner)
logging.info("All workers are running.")
logging.info(
"Killing the agent. All child processes including the worker have to be terminated"
)
process.kill()
process.wait()
# loop through all children until all of them are terminated.
# in case of failure the tests case timeout will be raised.
while True:
time.sleep(1)
# wait more if the agent is not killed yet.
if process.is_running():
logging.info("The agent process is still alive.")
continue
for child in children:
if child.is_running():
logging.info(
"The child process with pid {0} is still alive.".format(process.pid)
)
break
else:
# all child processes are terminated.
break
logging.info("All children are terminated.")
_check_workers_gracefull_stop(runner, worker_pids)
def test_standalone_agent_stop():
runner = AgentRunner(
DEV_INSTALL,
enable_debug_log=True,
workers_type="process",
workers_session_count=2,
)
runner.start()
logging.info("Agent started.")
time.sleep(1)
process, worker_pids, children, workers_processes = _perform_workers_check(runner)
_check_workers_gracefull_stop(runner, worker_pids, occurrences=0)
_stop_and_perform_checks(runner)
_check_workers_gracefull_stop(runner, worker_pids, occurrences=1)
def common_test_only_python_mapped_to_python3(install_package_fn,
install_next_version_fn):
"""
Test package installation on the machine with python3 but there is only 'python' command which is mapped on to it.
:param install_package_fn: callable that installs package with appropriate type to the current machine OS.
"""
# map 'python' command on to python3
_link_to_default_python("python3")
_remove_python("python2")
_remove_python("python3")
stdout, _ = install_package_fn()
# make sure that installer has found 'python' mapped on to python3
# this is signaled by the install script not trying to switch the interpreter
assert "Switching the Python interpreter used by the Scalyr Agent" not in stdout
# 'scalyr-agent-2-config' command must be a symlink to config_main.py
assert _get_current_config_script_name() == "config_main.py"
runner = AgentRunner(PACKAGE_INSTALL)
runner.start()
time.sleep(1)
assert _get_python_major_version(runner) == 3
runner.stop()
# install next version of the package
install_next_version_fn()
# the source file should be "config_main.py"
assert _get_current_config_script_name() == "config_main.py"
def _test_standalone_smoke(agent_installation_type, python_version=None):
"""
Agent standalone test to run within the same machine.
"""
if agent_installation_type == DEV_INSTALL:
os.chdir(get_package_root())
print("Agent host name: {0}".format(
compat.os_environ_unicode["AGENT_HOST_NAME"]))
runner = AgentRunner(agent_installation_type, enable_debug_log=True)
if python_version:
runner.switch_version(python_version)
agent_log_verifier = AgentLogVerifier(
runner, compat.os_environ_unicode["SCALYR_SERVER"])
data_json_verifier = DataJsonVerifier(
runner, compat.os_environ_unicode["SCALYR_SERVER"])
system_metrics_verifier = SystemMetricsVerifier(
runner, compat.os_environ_unicode["SCALYR_SERVER"])
process_metrics_verifier = ProcessMetricsVerifier(
runner, compat.os_environ_unicode["SCALYR_SERVER"])
_create_data_json_file(runner=runner,
data_json_verifier=data_json_verifier)
runner.start()
time.sleep(1)
print("Verify 'agent.log'")
assert agent_log_verifier.verify(
), "Verification of the file: 'agent.log' failed"
print("Verify 'linux_system_metrics.log'")
assert (system_metrics_verifier.verify()
), "Verification of the file: 'linux_system_metrics.log' failed"
print("Verify 'linux_process_metrics.log'")
assert (process_metrics_verifier.verify()
), "Verification of the file: 'linux_process_metrics.log' failed"
assert data_json_verifier.verify(
), "Verification of the file: 'data.json' failed"
runner.stop()
def test_default_compression_algorithm(request):
runner = AgentRunner(PACKAGE_INSTALL, send_to_server=False)
# deflate is the default
install_rpm()
runner.start()
time.sleep(1)
agent_status = runner.status_json(True)
compression_algorithm = agent_status["compression_type"]
compression_level = agent_status["compression_level"]
assert compression_algorithm == "deflate", "expected deflate, got %s" % (
compression_algorithm)
assert compression_level == 6
runner.stop()
def test_standalone_agent_restart():
runner = AgentRunner(
DEV_INSTALL,
enable_debug_log=True,
workers_type="process",
workers_session_count=2,
)
runner.start()
logging.info("Agent started.")
time.sleep(1)
process, worker_pids, children, workers_processes = _perform_workers_check(runner)
_check_workers_gracefull_stop(runner, worker_pids, occurrences=0)
runner.restart()
time.sleep(1)
assert (
not process.is_running()
), "the agent's previous process should terminated after the restart."
for worker in workers_processes:
assert (
not worker.is_running()
), "the previous worker processes should be terminated."
for child in children:
assert (
not child.is_running()
), "the child processed should be terminated after the restart."
process2, worker_pids2, children2, workers_processes2 = _perform_workers_check(
runner
)
assert process.pid != process2.pid
_check_workers_gracefull_stop(runner, worker_pids, occurrences=1)
_stop_and_perform_checks(runner)
_check_workers_gracefull_stop(runner, worker_pids2, occurrences=2)
def common_test_python2(install_package_fn, install_next_version_fn):
"""
Test package installation on machine with python2.
The function also verifies correct rc*.d symlinks are created.
:param install_package_fn: callable that installs package with appropriate type to the current machine OS.
"""
_remove_python("python")
_remove_python("python3")
# rc.d symlinks shouldn't exist before install
files = _get_agent_rc_d_symlinks()
assert len(files) == 0
stdout, _ = install_package_fn()
# But they should have been created during the postinst step
_assert_rc_d_symlinks_exist()
# make sure that installer has found 'python2'.
assert "The default 'python' command not found, will use python2 binary" in stdout
# 'scalyr-agent-2-config' command must be a symlink to config_main_py2.py
assert _get_current_config_script_name() == "config_main_py2.py"
runner = AgentRunner(PACKAGE_INSTALL)
runner.start()
time.sleep(1)
assert _get_python_major_version(runner) == 2
runner.stop()
# install next version of the package
stdout, _ = install_next_version_fn()
# the source file should be "config_main_py2.py"
assert _get_current_config_script_name() == "config_main_py2.py"
def test_standalone_agent_config_reload():
runner = AgentRunner(
DEV_INSTALL,
enable_debug_log=True,
workers_type="process",
workers_session_count=2,
)
runner.start()
logging.info("Agent started.")
time.sleep(1)
process, worker_pids, children, workers_processes = _perform_workers_check(runner)
_check_workers_gracefull_stop(runner, worker_pids, occurrences=0)
logging.info(
"Increase the worker count in the config and wait for agent config reload."
)
old_worker_pids = worker_pids.copy()
config = runner.config
config_worker_count = config["default_sessions_per_worker"]
config["default_sessions_per_worker"] = config_worker_count + 1
runner.write_config(config)
logging.info("checking in the loop until all old worker processes are gone")
while True:
assert process.is_running(), "The agent process must not be terminated."
children_ids = [p.pid for p in process.children()]
still_running_old_ids = []
for old_id in old_worker_pids:
if old_id in children_ids:
still_running_old_ids.append(old_id)
if not still_running_old_ids:
logging.info("All old worker processes are terminated.")
break
logging.info(
"Wait for old worker processes are terminated. Now alive: {0}".format(
still_running_old_ids
)
)
time.sleep(1)
time.sleep(1)
logging.info("checking in loop until the workers number is increased")
while True:
try:
process, worker_pids, children, workers_processes = _perform_workers_check(
runner
)
assert process.is_running()
assert len(workers_processes) == config_worker_count + 1
except:
time.sleep(1)
else:
break
_check_workers_gracefull_stop(runner, old_worker_pids, occurrences=1)
_stop_and_perform_checks(runner)
_check_workers_gracefull_stop(runner, old_worker_pids, occurrences=2)
new_workers = {}
for worker_id, worker_pid in worker_pids.items():
if worker_id not in old_worker_pids:
new_workers[worker_id] = worker_pid
_check_workers_gracefull_stop(runner, new_workers, occurrences=1)
def _test_standalone_smoke(
agent_installation_type,
python_version=None,
rate_limited=False,
workers_type="thread",
workers_sessions_count=1,
):
"""
Agent standalone test to run within the same machine.
"""
if agent_installation_type == DEV_INSTALL:
os.chdir(get_package_root())
print("Agent host name: {0}".format(compat.os_environ_unicode["AGENT_HOST_NAME"]))
runner = AgentRunner(
agent_installation_type,
enable_debug_log=True,
workers_type=workers_type,
workers_session_count=workers_sessions_count,
)
if python_version:
runner.switch_version(python_version)
agent_log_verifier = AgentLogVerifier(
runner, compat.os_environ_unicode["SCALYR_SERVER"]
)
worker_sessions_verifier = None
if workers_type == "process":
worker_sessions_verifier = AgentWorkerSessionLogVerifier(
runner, compat.os_environ_unicode["SCALYR_SERVER"]
)
if rate_limited:
data_json_verifier = DataJsonVerifierRateLimited(
runner, compat.os_environ_unicode["SCALYR_SERVER"]
)
else:
data_json_verifier = DataJsonVerifier(
runner, compat.os_environ_unicode["SCALYR_SERVER"]
)
system_metrics_verifier = SystemMetricsVerifier(
runner, compat.os_environ_unicode["SCALYR_SERVER"]
)
process_metrics_verifier = ProcessMetricsVerifier(
runner, compat.os_environ_unicode["SCALYR_SERVER"]
)
_create_data_json_file(runner=runner, data_json_verifier=data_json_verifier)
runner.start()
time.sleep(1)
print("Verify 'agent.log'")
assert agent_log_verifier.verify(), "Verification of the file: 'agent.log' failed"
if workers_type == "process":
# if workers are multiprocess, then we check if worker session logs are ingested.
print("Verify worker sessions log files.")
assert (
worker_sessions_verifier.verify()
), "Verification of the worker session log files is failed"
print("Verify 'linux_system_metrics.log'")
assert (
system_metrics_verifier.verify()
), "Verification of the file: 'linux_system_metrics.log' failed"
print("Verify 'linux_process_metrics.log'")
assert (
process_metrics_verifier.verify()
), "Verification of the file: 'linux_process_metrics.log' failed"
assert data_json_verifier.verify(), "Verification of the file: 'data.json' failed"
if workers_type == "process":
# increase worker sessions count, restart and also check if all worker logs are ingested.
config = runner.config
sessions_count = config["default_sessions_per_worker"]
sessions_count += 1
worker_sessions_verifier = AgentWorkerSessionLogVerifier(
runner, compat.os_environ_unicode["SCALYR_SERVER"]
)
config["default_sessions_per_worker"] = sessions_count
runner.write_config(config)
runner.restart()
agent_status = json.loads(runner.status_json())
assert (
len(agent_status["copying_manager_status"]["workers"][-1]["sessions"])
== sessions_count
)
print("Verify worker sessions log files.")
assert (
worker_sessions_verifier.verify()
), "Verification of the worker session log files is failed"
runner.stop()
def common_test_switch_python2_to_python3(install_package_fn,
install_next_version_fn):
"""
Test package installation on machine with both python2 and python3 but without default 'python'.
Package installer should pick python2 by default and then we switch to the python3.
:param install_package_fn: callable that installs package with appropriate type to the current machine OS.
"""
_remove_python("python")
install_package_fn()
assert _get_current_config_script_name() == "config_main_py2.py"
runner = AgentRunner(PACKAGE_INSTALL)
runner.start()
time.sleep(1)
assert _get_python_major_version(runner) == 2
# switching to python3
runner.stop()
runner.switch_version("python3")
assert _get_current_config_script_name() == "config_main_py3.py"
runner.start()
time.sleep(1)
assert _get_python_major_version(runner) == 3
runner.stop()
# install next version of the package and check that links are the same.
stdout, _ = install_next_version_fn()
# Installer must not switch to default python.
assert "Switching the Python interpreter used by the Scalyr Agent" not in stdout
# the source file should be "config_main_py3.py"
assert _get_current_config_script_name() == "config_main_py3.py"
# switching bach to python2
runner.switch_version("python2")
assert _get_current_config_script_name() == "config_main_py2.py"
runner.start()
time.sleep(1)
assert _get_python_major_version(runner) == 2
def common_test_switch_command_works_without_agent_config(install_package_fn):
"""
Verify that the Python version switch command works if the config file is not present or
doesn't contain a valid API key.
This is important because this command may run as part of package postinstall script when the
key won't be configured yet.
Basically it asserts that the switch command doesn't rely agent config being present and
correctly configured.
"""
install_package_fn()
runner = AgentRunner(PACKAGE_INSTALL)
# Make sure the config is not present
agent_config_path = "/etc/scalyr-agent-2/agent.json"
agent_config_d_path = "/etc/scalyr-agent-2/agent.d"
if os.path.isfile(agent_config_path):
os.unlink(agent_config_path)
if os.path.isdir(agent_config_d_path):
shutil.rmtree(agent_config_d_path)
# Make sure no SCALYR_ environment variables are set
env = compat.os_environ_unicode.copy()
for key in list(env.keys()):
if key.lower().startswith("scalyr"):
del env[key]
binary_path = os.path.join("/", "usr", "share", "scalyr-agent-2", "bin")
scalyr_agent_2_target = os.path.join(binary_path, "scalyr-agent-2")
scalyr_agent_2_config_target = os.path.join(binary_path,
"scalyr-agent-2-config")
# Default should be python binary
shebang_line_main = get_shebang_from_file(scalyr_agent_2_target)
shebang_line_config = get_shebang_from_file(scalyr_agent_2_config_target)
# On some newer distros python binary is not available
if shutil.which("python"):
is_python_binary_available = True
expected = "#!/usr/bin/env python"
else:
is_python_binary_available = False
expected = "#!/usr/bin/env python2"
assert shebang_line_main == expected, "expected %s, got %s" % (
expected,
shebang_line_main,
)
assert shebang_line_config == expected, "expected %s, got %s" % (
expected,
shebang_line_config,
)
# Switch to python3
runner.switch_version("python3", env=env)
shebang_line_main = get_shebang_from_file(scalyr_agent_2_target)
shebang_line_config = get_shebang_from_file(scalyr_agent_2_config_target)
assert shebang_line_main == "#!/usr/bin/env python3"
assert shebang_line_config == "#!/usr/bin/env python3"
# Switch back to python2
runner.switch_version("python2", env=env)
shebang_line_main = get_shebang_from_file(scalyr_agent_2_target)
shebang_line_config = get_shebang_from_file(scalyr_agent_2_config_target)
assert shebang_line_main == "#!/usr/bin/env python2"
assert shebang_line_config == "#!/usr/bin/env python2"
# Switch back to python (aka default)
if is_python_binary_available:
runner.switch_version("python", env=env)
shebang_line_main = get_shebang_from_file(scalyr_agent_2_target)
shebang_line_config = get_shebang_from_file(
scalyr_agent_2_config_target)
assert shebang_line_main == "#!/usr/bin/env python"
assert shebang_line_config == "#!/usr/bin/env python"
# Write a config with invalid config, this way we ensure config is indeed not parsed by that
# command even if it's present
mock_config = {
"api_key": "",
"scalyr_server": "agent.scalyr.com",
}
with open(agent_config_path, "w") as fp:
fp.write(json.dumps(mock_config))
# Switch to python3
runner.switch_version("python3", env=env)
shebang_line_main = get_shebang_from_file(scalyr_agent_2_target)
shebang_line_config = get_shebang_from_file(scalyr_agent_2_config_target)
assert shebang_line_main == "#!/usr/bin/env python3"
assert shebang_line_config == "#!/usr/bin/env python3"
def common_test_switch_default_to_python3(install_package_fn,
install_next_version_fn):
"""
Test package installation on machine with python and python3.
Package installer should pick 'python' by default and then we switch it to python3.
:param install_package_fn: callable that installs package with appropriate type to the current machine OS.
"""
# let python2 be default.
_link_to_default_python("python2")
_remove_python("python2")
stdout, _ = install_package_fn()
# make sure that installer has found 'python' mapped on to python3.
# this is signaled by the install script not trying to switch the interpreter
assert "Switching the Python interpreter used by the Scalyr Agent" not in stdout
assert _get_current_config_script_name() == "config_main.py"
runner = AgentRunner(PACKAGE_INSTALL)
runner.start()
time.sleep(1)
assert _get_python_major_version(runner) == 2
# switching to python3
runner.stop()
runner.switch_version("python3")
assert _get_current_config_script_name() == "config_main_py3.py"
runner.start()
time.sleep(1)
assert _get_python_major_version(runner) == 3
runner.stop()
# install next version of the package and check that links are the same.
stdout, _ = install_next_version_fn()
# Installer must not switch to default python.
assert "Switching the Python interpreter used by the Scalyr Agent" not in stdout
# the source file should be "config_main_py3.py"
assert _get_current_config_script_name() == "config_main_py3.py"
# switching back to default python
runner.switch_version("default")
assert _get_current_config_script_name() == "config_main.py"
runner.start()
time.sleep(1)
assert _get_python_major_version(runner) == 2
