def run_puppet_agent(base_image, init_system):
dockerfile = os.path.join(DOCKERFILES_DIR, "Dockerfile.%s" % base_image)
with run_init_system_image(base_image,
path=REPO_ROOT_DIR,
dockerfile=dockerfile,
with_socat=False) as [
cont,
backend,
]:
if (base_image, init_system) in DEB_DISTROS:
code, output = cont.exec_run(
f"puppet module install puppetlabs-apt --version {APT_MODULE_VERSION}"
)
assert code == 0, output.decode("utf-8")
print_lines(output)
with tempfile.NamedTemporaryFile(mode="w") as fd:
fd.write(
CONFIG.substitute(ingest_url=backend.ingest_url,
api_url=backend.api_url))
fd.flush()
copy_file_into_container(fd.name, cont, "/root/agent.pp")
code, output = cont.exec_run("puppet apply /root/agent.pp")
assert code in (0, 2), output.decode("utf-8")
print_lines(output)
try:
yield cont, backend
finally:
print("Agent log:")
print_lines(get_agent_logs(cont, init_system))
def _run_tests(base_image,
init_system,
installer_args,
user=None,
**extra_run_kwargs):
if user:
installer_args = f"--service-user {user} --service-group {user} {installer_args}"
else:
user = "******"
with run_init_system_image(base_image,
**extra_run_kwargs) as [cont, backend]:
copy_file_into_container(INSTALLER_PATH, cont, "/opt/install.sh")
# Unfortunately, wget and curl both don't like self-signed certs, even
# if they are in the system bundle, so we need to use the --insecure
# flag.
code, output = cont.exec_run(
f"sh /opt/install.sh --insecure {installer_args}")
print("Output of install script:")
print_lines(output)
assert code == 0, "Agent could not be installed!"
try:
verify_override_files(cont, init_system, user)
assert is_agent_running_as_non_root(
cont, user), f"Agent is not running as {user} user"
yield backend, cont
finally:
print("Agent log:")
print_lines(get_agent_logs(cont, init_system))
def run_ansible(cont,
init_system,
backend,
monitors,
agent_version,
stage,
user="******"):
with tempfile.NamedTemporaryFile(mode="w+") as fd:
config_yaml = get_config(backend, monitors, agent_version, stage, user)
print(config_yaml)
fd.write(config_yaml)
fd.flush()
copy_file_into_container(fd.name, cont, CONFIG_DEST_PATH)
code, output = cont.exec_run(ANSIBLE_CMD)
assert code == 0, output.decode("utf-8")
print_lines(output)
verify_override_files(cont, init_system, user)
installed_version = get_agent_version(cont).replace("~", "-")
agent_version = re.sub(r"-\d+$", "", agent_version).replace("~", "-")
assert installed_version == agent_version, "installed agent version is '%s', expected '%s'" % (
installed_version,
agent_version,
)
assert is_agent_running_as_non_root(
cont, user=user), f"Agent is not running as {user} user"
def run_puppet_agent(cont,
init_system,
backend,
monitors,
agent_version,
stage,
user="******"):
with tempfile.NamedTemporaryFile(mode="w+") as fd:
hiera_yaml = get_hiera(HIERA_SRC_PATH, backend, monitors)
print(hiera_yaml)
fd.write(hiera_yaml)
fd.flush()
copy_file_into_container(fd.name, cont, HIERA_DEST_PATH)
with tempfile.NamedTemporaryFile(mode="w+") as fd:
config = get_config(agent_version, stage, user=user)
print(config)
fd.write(config)
fd.flush()
copy_file_into_container(fd.name, cont, "/root/agent.pp")
code, output = cont.exec_run("puppet apply /root/agent.pp")
assert code in (0, 2), output.decode("utf-8")
print_lines(output)
verify_override_files(cont, init_system, user)
installed_version = get_agent_version(cont)
assert installed_version == agent_version, "installed agent version is '%s', expected '%s'" % (
installed_version,
agent_version,
)
assert is_agent_running_as_non_root(
cont, user=user), f"Agent is not running as {user} user"
def run_puppet_apply(container, config):
with tempfile.NamedTemporaryFile(mode="w+") as fd:
print(config)
fd.write(config)
fd.flush()
copy_file_into_container(container, fd.name, "/root/test.pp")
run_container_cmd(container, "puppet apply /root/test.pp")
def run_salt_apply(container, config):
with tempfile.NamedTemporaryFile(mode="w+") as fd:
print(config)
fd.write(config)
fd.flush()
copy_file_into_container(container, fd.name, PILLAR_PATH)
run_container_cmd(container, SALT_CMD)
def test_installer_without_fluentd(distro, version):
install_cmd = f"sh -x /test/install.sh -- testing123 --realm us0 --memory {TOTAL_MEMORY} --without-fluentd"
if version != "latest":
install_cmd = f"{install_cmd} --collector-version {version.lstrip('v')}"
if STAGE != "release":
assert STAGE in ("test", "beta"), f"Unsupported stage '{STAGE}'!"
install_cmd = f"{install_cmd} --{STAGE}"
print(f"Testing installation on {distro} from {STAGE} stage ...")
with run_distro_container(distro) as container:
copy_file_into_container(container, INSTALLER_PATH, "/test/install.sh")
try:
# run installer script
run_container_cmd(container,
install_cmd,
env={"VERIFY_ACCESS_TOKEN": "false"})
time.sleep(5)
config_path = AGENT_CONFIG_PATH
if container.exec_run(f"test -f {OLD_CONFIG_PATH}").exit_code == 0:
config_path = OLD_CONFIG_PATH
splunk_env_path = SPLUNK_ENV_PATH
if container.exec_run(
f"test -f {OLD_SPLUNK_ENV_PATH}").exit_code == 0:
splunk_env_path = OLD_SPLUNK_ENV_PATH
# verify env file created with configured parameters
run_container_cmd(
container,
f"grep '^SPLUNK_CONFIG={config_path}$' {splunk_env_path}")
run_container_cmd(
container,
f"grep '^SPLUNK_ACCESS_TOKEN=testing123$' {splunk_env_path}")
run_container_cmd(container,
f"grep '^SPLUNK_REALM=us0$' {splunk_env_path}")
run_container_cmd(
container,
f"grep '^SPLUNK_MEMORY_TOTAL_MIB={TOTAL_MEMORY}$' {splunk_env_path}"
)
# verify collector service status
assert wait_for(lambda: service_is_running(
container, service_owner=SERVICE_OWNER))
if distro in DEB_DISTROS:
assert container.exec_run("dpkg -s td-agent").exit_code != 0
else:
assert container.exec_run("rpm -q td-agent").exit_code != 0
run_container_cmd(container, "sh -x /test/install.sh --uninstall")
finally:
run_container_cmd(container,
f"journalctl -u {SERVICE_NAME} --no-pager")
def _test_package_upgrade(base_image, package_path, init_system):
with run_init_system_image(base_image, with_socat=False) as [cont, backend]:
_, package_ext = os.path.splitext(package_path)
copy_file_into_container(package_path, cont, "/opt/signalfx-agent%s" % package_ext)
install_deps(cont, base_image)
if "opensuse" in base_image:
install_cmd = f"bash -ec 'wget -nv {OLD_SUSE_RPM_URL} && rpm -ivh --nodeps {OLD_SUSE_RPM_NAME}'"
upgrade_cmd = "rpm -Uvh --nodeps /opt/signalfx-agent.rpm"
else:
install_cmd = OLD_INSTALL_COMMAND[package_ext]
upgrade_cmd = UPGRADE_COMMAND[package_ext]
_test_install_package(cont, install_cmd)
cont.exec_run("bash -ec 'echo -n testing123 > /etc/signalfx/token'")
old_agent_yaml = update_agent_yaml(cont, backend, hostname="test-" + base_image)
_, _ = cont.exec_run("cp -f %s %s.orig" % (AGENT_YAML_PATH, AGENT_YAML_PATH))
code, output = cont.exec_run(upgrade_cmd)
print("Output of package upgrade:")
print_lines(output)
assert code == 0, "Package could not be upgraded!"
_test_package_verify(cont, package_ext, base_image)
if init_system == INIT_SYSTEMD:
assert not path_exists_in_container(cont, "/etc/init.d/signalfx-agent")
else:
assert path_exists_in_container(cont, "/etc/init.d/signalfx-agent")
new_agent_yaml = get_container_file_content(cont, AGENT_YAML_PATH)
diff = get_agent_yaml_diff(old_agent_yaml, new_agent_yaml)
assert not diff, "%s different after upgrade!\n%s" % (AGENT_YAML_PATH, diff)
try:
_test_service_list(cont, init_system)
_test_service_restart(cont, init_system, backend)
_test_service_status(cont, init_system, "active")
_test_service_stop(cont, init_system, backend)
_test_service_status(cont, init_system, "inactive")
_test_service_start(cont, init_system, backend)
_test_service_status(cont, init_system, "active")
_test_service_stop(cont, init_system, backend)
_test_system_restart(cont, init_system, backend)
if init_system == INIT_SYSTEMD:
_test_service_status_redirect(cont)
_test_agent_status(cont)
_test_service_override(cont, init_system, backend)
finally:
cont.reload()
if cont.status.lower() == "running":
print("Agent service status:")
print_lines(cont.exec_run(INIT_STATUS_COMMAND[init_system]).output)
print("Agent log:")
print_lines(get_agent_logs(cont, init_system))
def test_helm(k8s_cluster, helm_version):
helm_major_version = int(helm_version.split(".")[0])
with run_helm_image(k8s_cluster, helm_version) as cont:
with k8s_cluster.create_resources(
[APP_YAML_PATH]), tiller_rbac_resources(
k8s_cluster,
helm_major_version), fake_backend.start() as backend:
if helm_major_version < 3:
init_helm(k8s_cluster, cont, helm_major_version)
with k8s_cluster.run_tunnels(backend) as proxy_pod_ip:
with release_values_yaml(k8s_cluster, proxy_pod_ip,
backend) as values_path:
copy_file_into_container(values_path, cont, values_path)
install_helm_chart(k8s_cluster, values_path, cont,
helm_major_version)
try:
assert wait_for(
p(
has_datapoint,
backend,
dimensions={
"container_name": "prometheus",
"application": "helm-test"
},
),
timeout_seconds=60,
)
assert wait_for(p(
has_datapoint,
backend,
dimensions={"plugin": "signalfx-metadata"}),
timeout_seconds=60)
finally:
for pod in get_pods_by_labels(
"app=signalfx-agent",
namespace=k8s_cluster.test_namespace):
print("pod/%s:" % pod.metadata.name)
status = exec_pod_command(
pod.metadata.name,
AGENT_STATUS_COMMAND,
namespace=k8s_cluster.test_namespace)
print("Agent Status:\n%s" % status)
logs = get_pod_logs(
pod.metadata.name,
namespace=k8s_cluster.test_namespace)
print("Agent Logs:\n%s" % logs)
print("\nDatapoints received:")
for dp in backend.datapoints:
print_dp_or_event(dp)
print("\nEvents received:")
for event in backend.events:
print_dp_or_event(event)
print(f"\nDimensions set: {backend.dims}")
def _test_package_upgrade(base_image, package_path, init_system):
with run_init_system_image(base_image,
with_socat=False) as [cont, backend]:
_, package_ext = os.path.splitext(package_path)
copy_file_into_container(package_path, cont,
"/opt/signalfx-agent%s" % package_ext)
code, output = cont.exec_run(OLD_INSTALL_COMMAND[package_ext])
print("Output of old package install:")
print_lines(output)
assert code == 0, "Old package could not be installed!"
cont.exec_run("bash -ec 'echo -n testing123 > /etc/signalfx/token'")
old_agent_yaml = update_agent_yaml(cont,
backend,
hostname="test-" + base_image)
_, _ = cont.exec_run("cp -f %s %s.orig" %
(AGENT_YAML_PATH, AGENT_YAML_PATH))
code, output = cont.exec_run(UPGRADE_COMMAND[package_ext])
print("Output of package upgrade:")
print_lines(output)
assert code == 0, "Package could not be upgraded!"
if init_system == INIT_SYSTEMD:
assert not path_exists_in_container(cont,
"/etc/init.d/signalfx-agent")
else:
assert path_exists_in_container(cont, "/etc/init.d/signalfx-agent")
new_agent_yaml = get_container_file_content(cont, AGENT_YAML_PATH)
diff = get_agent_yaml_diff(old_agent_yaml, new_agent_yaml)
assert not diff, "%s different after upgrade!\n%s" % (AGENT_YAML_PATH,
diff)
try:
_test_service_list(cont, init_system)
_test_service_restart(cont, init_system, backend)
_test_service_status(cont, init_system, "active")
_test_service_stop(cont, init_system, backend)
_test_service_status(cont, init_system, "inactive")
_test_service_start(cont, init_system, backend)
_test_service_status(cont, init_system, "active")
_test_service_stop(cont, init_system, backend)
_test_system_restart(cont, init_system, backend)
if init_system == INIT_SYSTEMD:
_test_service_status_redirect(cont)
_test_agent_status(cont)
finally:
cont.reload()
if cont.status.lower() == "running":
print("Agent log:")
print_lines(get_agent_logs(cont, init_system))
def test_collector_package_upgrade(distro):
install_cmd = f"sh /test/install.sh -- testing123 --realm test --without-fluentd --collector-version 0.35.0"
pkg_path = get_package(distro, PKG_NAME, PKG_DIR)
assert pkg_path, f"{PKG_NAME} package not found in {PKG_DIR}"
pkg_base = os.path.basename(pkg_path)
with run_distro_container(distro) as container:
copy_file_into_container(container, INSTALLER_PATH, "/test/install.sh")
try:
# install an older version of the collector package
run_container_cmd(container,
install_cmd,
env={"VERIFY_ACCESS_TOKEN": "false"})
time.sleep(5)
# verify collector service status
assert wait_for(lambda: service_is_running(
container, service_owner=SERVICE_OWNER))
# change the config
run_container_cmd(
container,
f"sh -c 'echo \"# This line should be preserved\" >> {AGENT_CONFIG_PATH}'"
)
copy_file_into_container(container, pkg_path, f"/test/{pkg_base}")
# upgrade package
if distro in DEB_DISTROS:
run_container_cmd(container,
f"dpkg -i --force-confold /test/{pkg_base}")
else:
run_container_cmd(container, f"rpm -U /test/{pkg_base}")
time.sleep(5)
# verify collector service status
assert wait_for(lambda: service_is_running(
container, service_owner=SERVICE_OWNER))
# verify changed config was preserved after upgrade
run_container_cmd(
container,
f"grep '# This line should be preserved' {AGENT_CONFIG_PATH}")
finally:
run_container_cmd(container,
f"journalctl -u {SERVICE_NAME} --no-pager")
def test_installer(distro, version, memory_option):
install_cmd = f"sh -x /test/install.sh -- testing123 --realm us0"
if memory_option == "memory":
install_cmd = f"{install_cmd} --{memory_option} {TOTAL_MEMORY}"
elif memory_option == "ballast":
install_cmd = f"{install_cmd} --{memory_option} {BALLAST}"
if version != "latest":
install_cmd = f"{install_cmd} --collector-version {version.lstrip('v')}"
if STAGE != "release":
assert STAGE in ("test", "beta"), f"Unsupported stage '{STAGE}'!"
install_cmd = f"{install_cmd} --{STAGE}"
print(f"Testing installation on {distro} from {STAGE} stage ...")
with run_distro_container(distro) as container:
# run installer script
copy_file_into_container(container, INSTALLER_PATH, "/test/install.sh")
try:
run_container_cmd(container, install_cmd, env={"VERIFY_ACCESS_TOKEN": "false"})
time.sleep(5)
# verify env file created with configured parameters
splunk_env_path = SPLUNK_ENV_PATH
if container.exec_run(f"test -f {OLD_SPLUNK_ENV_PATH}").exit_code == 0:
splunk_env_path = OLD_SPLUNK_ENV_PATH
run_container_cmd(container, f"grep '^SPLUNK_ACCESS_TOKEN=testing123$' {splunk_env_path}")
run_container_cmd(container, f"grep '^SPLUNK_REALM=us0$' {splunk_env_path}")
if memory_option == "memory":
run_container_cmd(container, f"grep '^SPLUNK_MEMORY_TOTAL_MIB={TOTAL_MEMORY}$' {splunk_env_path}")
elif memory_option == "ballast":
run_container_cmd(container, f"grep '^SPLUNK_BALLAST_SIZE_MIB={BALLAST}$' {splunk_env_path}")
# verify collector service status
assert wait_for(lambda: service_is_running(container, service_owner=SERVICE_OWNER))
# the td-agent service should only be running when installing
# collector packages that have our custom fluent config
if container.exec_run("test -f /etc/otel/collector/fluentd/fluent.conf").exit_code == 0:
assert container.exec_run("systemctl status td-agent").exit_code == 0
else:
assert container.exec_run("systemctl status td-agent").exit_code != 0
run_container_cmd(container, "sh -x /test/install.sh --uninstall")
finally:
run_container_cmd(container, "journalctl -u td-agent --no-pager")
if container.exec_run("test -f /var/log/td-agent/td-agent.log").exit_code == 0:
run_container_cmd(container, "cat /var/log/td-agent/td-agent.log")
run_container_cmd(container, f"journalctl -u {SERVICE_NAME} --no-pager")
def run_puppet_agent(cont, backend, monitors, agent_version, stage):
with tempfile.NamedTemporaryFile(mode="w+") as fd:
fd.write(get_config(backend, monitors, agent_version, stage))
fd.flush()
copy_file_into_container(fd.name, cont, "/root/agent.pp")
code, output = cont.exec_run("puppet apply /root/agent.pp")
assert code in (0, 2), output.decode("utf-8")
print_lines(output)
installed_version = get_agent_version(cont)
assert installed_version == agent_version, "installed agent version is '%s', expected '%s'" % (
installed_version,
agent_version,
)
assert is_agent_running_as_non_root(cont), "Agent is not running as non-root user"
def test_bundle(request, base_image):
# Get bundle path from command line flag to pytest
bundle_path = request.config.getoption("--test-bundle-path")
if not bundle_path:
raise ValueError(
"You must specify the --test-bundle-path flag to run bundle tests")
with run_service("activemq") as activemq_container:
with run_init_system_image(base_image,
command="/usr/bin/tail -f /dev/null") as [
cont, backend
]:
copy_file_into_container(bundle_path, cont, "/opt/bundle.tar.gz")
code, output = cont.exec_run(f"tar -xf /opt/bundle.tar.gz -C /opt")
assert code == 0, f"Could not untar bundle: {output}"
code, output = cont.exec_run(
f"/opt/signalfx-agent/bin/patch-interpreter /opt/signalfx-agent"
)
assert code == 0, f"Could not patch interpreter: {output}"
write_agent_config(cont, activemq_container)
_, output = cont.exec_run(
[
"/bin/sh", "-c",
"exec /opt/signalfx-agent/bin/signalfx-agent > /var/log/signalfx-agent.log"
],
detach=True,
stream=True,
)
try:
assert wait_for(
p(has_datapoint, backend, metric_name="cpu.utilization"),
timeout_seconds=10
), "Python metadata datapoint didn't come through"
assert wait_for(
p(has_datapoint,
backend,
metric_name="gauge.amq.queue.QueueSize")
), "Didn't get activemq queue size datapoint"
code, output = cont.exec_run(
"/opt/signalfx-agent/bin/agent-status")
assert code == 0, f"failed to execute agent-status:\n{output.decode('utf-8')}"
finally:
print("Agent log:")
_, output = cont.exec_run("cat /var/log/signalfx-agent.log")
print_lines(output)
def _test_package_install(base_image, package_path, init_system):
with run_init_system_image(base_image,
with_socat=False) as [cont, backend]:
_, package_ext = os.path.splitext(package_path)
copy_file_into_container(package_path, cont,
"/opt/signalfx-agent%s" % package_ext)
if "opensuse" in base_image:
# suse-specific installation (this will usually be handled by the installer script)
code, output = cont.exec_run(
"zypper install -y -l libcap2 libcap-progs libpcap1 shadow")
assert code == 0, "Failed to install dependencies:\n%s" % output.decode(
"utf-8")
install_cmd = "rpm -ivh --nodeps /opt/signalfx-agent.rpm"
else:
install_cmd = INSTALL_COMMAND[package_ext]
code, output = cont.exec_run(install_cmd)
print("Output of package install:")
print_lines(output)
assert code == 0, "Package could not be installed!"
if init_system == INIT_SYSTEMD:
assert not path_exists_in_container(cont,
"/etc/init.d/signalfx-agent")
else:
assert path_exists_in_container(cont, "/etc/init.d/signalfx-agent")
cont.exec_run("bash -ec 'echo -n testing123 > /etc/signalfx/token'")
update_agent_yaml(cont, backend, hostname="test-" + base_image)
try:
_test_service_list(cont, init_system)
_test_service_restart(cont, init_system, backend)
_test_service_status(cont, init_system, "active")
_test_service_stop(cont, init_system, backend)
_test_service_status(cont, init_system, "inactive")
_test_service_start(cont, init_system, backend)
_test_service_status(cont, init_system, "active")
_test_service_stop(cont, init_system, backend)
_test_system_restart(cont, init_system, backend)
if init_system == INIT_SYSTEMD:
_test_service_status_redirect(cont)
_test_agent_status(cont)
finally:
cont.reload()
if cont.status.lower() == "running":
print("Agent log:")
print_lines(get_agent_logs(cont, init_system))
def _test_package_install(base_image, package_path, init_system):
with run_init_system_image(base_image,
with_socat=False) as [cont, backend]:
_, package_ext = os.path.splitext(package_path)
copy_file_into_container(package_path, cont,
"/opt/signalfx-agent%s" % package_ext)
install_deps(cont, base_image)
if "opensuse" in base_image:
install_cmd = "rpm -ivh --nodeps /opt/signalfx-agent.rpm"
else:
install_cmd = INSTALL_COMMAND[package_ext]
_test_install_package(cont, install_cmd)
_test_package_verify(cont, package_ext, base_image)
if init_system == INIT_SYSTEMD:
assert not path_exists_in_container(cont,
"/etc/init.d/signalfx-agent")
else:
assert path_exists_in_container(cont, "/etc/init.d/signalfx-agent")
cont.exec_run("bash -ec 'echo -n testing123 > /etc/signalfx/token'")
update_agent_yaml(cont, backend, hostname="test-" + base_image)
try:
_test_service_list(cont, init_system)
_test_service_restart(cont, init_system, backend)
_test_service_status(cont, init_system, "active")
_test_service_stop(cont, init_system, backend)
_test_service_status(cont, init_system, "inactive")
_test_service_start(cont, init_system, backend)
_test_service_status(cont, init_system, "active")
_test_service_stop(cont, init_system, backend)
_test_system_restart(cont, init_system, backend)
if init_system == INIT_SYSTEMD:
_test_service_status_redirect(cont)
_test_agent_status(cont)
_test_service_override(cont, init_system, backend)
finally:
cont.reload()
if cont.status.lower() == "running":
print("Agent service status:")
print_lines(
cont.exec_run(INIT_STATUS_COMMAND[init_system]).output)
print("Agent log:")
print_lines(get_agent_logs(cont, init_system))
def test_installer_mode(distro, version, mode):
install_cmd = f"sh -x /test/install.sh -- testing123 --realm {REALM} --memory {TOTAL_MEMORY} --mode {mode}"
if version != "latest":
install_cmd = f"{install_cmd} --collector-version {version.lstrip('v')}"
if STAGE != "release":
assert STAGE in ("test", "beta"), f"Unsupported stage '{STAGE}'!"
install_cmd = f"{install_cmd} --{STAGE}"
print(f"Testing installation on {distro} from {STAGE} stage ...")
with run_distro_container(distro) as container:
# run installer script
copy_file_into_container(container, INSTALLER_PATH, "/test/install.sh")
try:
run_container_cmd(container,
install_cmd,
env={"VERIFY_ACCESS_TOKEN": "false"})
time.sleep(5)
# verify env file created with configured parameters
verify_env_file(container, mode=mode)
# verify collector service status
assert wait_for(lambda: service_is_running(
container, service_owner=SERVICE_OWNER))
if "opensuse" not in distro:
assert container.exec_run(
"systemctl status td-agent").exit_code == 0
# test support bundle script
verify_support_bundle(container)
run_container_cmd(container, "sh -x /test/install.sh --uninstall")
finally:
if "opensuse" not in distro:
run_container_cmd(container,
"journalctl -u td-agent --no-pager")
if container.exec_run("test -f /var/log/td-agent/td-agent.log"
).exit_code == 0:
run_container_cmd(container,
"cat /var/log/td-agent/td-agent.log")
run_container_cmd(container,
f"journalctl -u {SERVICE_NAME} --no-pager")
def _run_tests(base_image, init_system, installer_args, **extra_run_kwargs):
with run_init_system_image(base_image, **extra_run_kwargs) as [cont, backend]:
copy_file_into_container(INSTALLER_PATH, cont, "/opt/install.sh")
# Unfortunately, wget and curl both don't like self-signed certs, even
# if they are in the system bundle, so we need to use the --insecure
# flag.
code, output = cont.exec_run(f"sh /opt/install.sh --insecure {installer_args}")
print("Output of install script:")
print_lines(output)
assert code == 0, "Agent could not be installed!"
try:
assert is_agent_running_as_non_root(cont), "Agent is running as root user"
yield backend, cont
finally:
print("Agent log:")
print_lines(get_agent_logs(cont, init_system))
def run_salt(cont, backend, agent_version, monitors, stage):
with tempfile.NamedTemporaryFile(mode="w+") as fd:
config_yaml = get_config(backend, agent_version, monitors, stage)
print(config_yaml)
fd.write(config_yaml)
fd.flush()
copy_file_into_container(fd.name, cont, PILLAR_PATH)
code, output = cont.exec_run(SALT_CMD)
print_lines(output)
assert code == 0, f"'{SALT_CMD}' failed"
installed_version = get_agent_version(cont)
assert installed_version == agent_version, "installed agent version is '%s', expected '%s'" % (
installed_version,
agent_version,
)
assert is_agent_running_as_non_root(
cont), "Agent is not running as non-root user"
def _test_package_install(base_image, package_path, init_system):
with run_init_system_image(base_image,
with_socat=False) as [cont, backend]:
_, package_ext = os.path.splitext(package_path)
copy_file_into_container(package_path, cont,
"/opt/signalfx-agent%s" % package_ext)
code, output = cont.exec_run(INSTALL_COMMAND[package_ext])
print("Output of package install:")
print_lines(output)
assert code == 0, "Package could not be installed!"
if init_system == INIT_SYSTEMD:
assert not path_exists_in_container(cont,
"/etc/init.d/signalfx-agent")
else:
assert path_exists_in_container(cont, "/etc/init.d/signalfx-agent")
cont.exec_run("bash -ec 'echo -n testing123 > /etc/signalfx/token'")
update_agent_yaml(cont, backend, hostname="test-" + base_image)
try:
_test_service_list(cont, init_system)
_test_service_restart(cont, init_system, backend)
_test_service_status(cont, init_system, "active")
_test_service_stop(cont, init_system, backend)
_test_service_status(cont, init_system, "inactive")
_test_service_start(cont, init_system, backend)
_test_service_status(cont, init_system, "active")
_test_service_stop(cont, init_system, backend)
_test_system_restart(cont, init_system, backend)
if init_system == INIT_SYSTEMD:
_test_service_status_redirect(cont)
finally:
cont.reload()
if cont.status.lower() == "running":
print("Agent log:")
print_lines(get_agent_logs(cont, init_system))
def test_bundle(request, base_image):
# Get bundle path from command line flag to pytest
bundle_path = request.config.getoption("--test-bundle-path")
if not bundle_path:
raise ValueError(
"You must specify the --test-bundle-path flag to run bundle tests")
with run_service("activemq") as activemq_container:
with run_init_system_image(base_image,
command="/usr/bin/tail -f /dev/null") as [
cont, backend
]:
copy_file_into_container(bundle_path, cont, "/opt/bundle.tar.gz")
code, output = cont.exec_run(f"tar -xf /opt/bundle.tar.gz -C /opt")
assert code == 0, f"Could not untar bundle: {output}"
code, output = cont.exec_run(
f"/opt/signalfx-agent/bin/patch-interpreter /opt/signalfx-agent"
)
assert code == 0, f"Could not patch interpreter: {output}"
agent = Agent(
host="127.0.0.1",
fake_services=None,
run_dir="/tmp/signalfx",
config=dedent(f"""
signalFxRealm: us0
observers:
- type: host
monitors:
- type: host-metadata
- type: cpu
- type: filesystems
- type: disk-io
- type: net-io
- type: load
- type: memory
- type: vmem
# This is a GenericJMX Java plugin, so we test the bundled Java runtime
- type: collectd/activemq
host: {container_ip(activemq_container)}
port: 1099
username: testuser
password: testing123
"""),
)
copy_file_content_into_container(agent.get_final_config_yaml(),
cont, "/etc/signalfx/agent.yaml")
_, output = cont.exec_run(
[
"/bin/sh", "-c",
"exec /opt/signalfx-agent/bin/signalfx-agent > /var/log/signalfx-agent.log"
],
detach=True,
stream=True,
)
try:
assert wait_for(
p(has_datapoint, backend, metric_name="cpu.utilization"),
timeout_seconds=10
), "Python metadata datapoint didn't come through"
assert wait_for(
p(has_datapoint,
backend,
metric_name="gauge.amq.queue.QueueSize")
), "Didn't get activemq queue size datapoint"
finally:
print("Agent log:")
_, output = cont.exec_run("cat /var/log/signalfx-agent.log")
print_lines(output)
def test_collector_package_install(distro):
pkg_name = "splunk-otel-collector"
pkg_dir = REPO_DIR / "dist"
service_name = "splunk-otel-collector"
service_owner = "splunk-otel-collector"
service_proc = "otelcol"
env_path = "/etc/otel/collector/splunk-otel-collector.conf"
agent_config_path = "/etc/otel/collector/agent_config.yaml"
gateway_config_path = "/etc/otel/collector/gateway_config.yaml"
bundle_dir = "/usr/lib/splunk-otel-collector/agent-bundle"
pkg_path = get_package(distro, pkg_name, pkg_dir)
assert pkg_path, f"{pkg_name} package not found in {pkg_dir}"
pkg_base = os.path.basename(pkg_path)
with run_distro_container(distro) as container:
# install setcap dependency
if distro in RPM_DISTROS:
if container.exec_run("command -v yum").exit_code == 0:
run_container_cmd(container, "yum install -y libcap")
else:
run_container_cmd(container, "dnf install -y libcap")
else:
run_container_cmd(container, "apt-get update")
run_container_cmd(container, "apt-get install -y libcap2-bin")
copy_file_into_container(container, pkg_path, f"/test/{pkg_base}")
try:
# install package
if distro in DEB_DISTROS:
run_container_cmd(container, f"dpkg -i /test/{pkg_base}")
else:
run_container_cmd(container, f"rpm -i /test/{pkg_base}")
run_container_cmd(container, f"test -d {bundle_dir}")
run_container_cmd(container, f"test -d {bundle_dir}/run/collectd")
run_container_cmd(container, f"test -f {agent_config_path}")
run_container_cmd(container, f"test -f {gateway_config_path}")
# verify service is not running after install without config file
time.sleep(5)
assert not service_is_running(container, service_name,
service_owner, service_proc)
# verify service starts with config file
run_container_cmd(container,
f"cp -f {env_path}.example {env_path}")
run_container_cmd(container, f"systemctl start {service_name}")
time.sleep(5)
assert wait_for(lambda: service_is_running(
container, service_name, service_owner, service_proc))
# verify service restart
run_container_cmd(container, f"systemctl restart {service_name}")
time.sleep(5)
assert wait_for(lambda: service_is_running(
container, service_name, service_owner, service_proc))
# verify service stop
run_container_cmd(container, f"systemctl stop {service_name}")
time.sleep(5)
assert not service_is_running(container, service_name,
service_owner, service_proc)
finally:
run_container_cmd(container,
f"journalctl -u {service_name} --no-pager")
# verify uninstall
run_container_cmd(container, f"systemctl start {service_name}")
time.sleep(5)
if distro in DEB_DISTROS:
run_container_cmd(container, f"dpkg -P {pkg_name}")
else:
run_container_cmd(container, f"rpm -e {pkg_name}")
time.sleep(5)
assert not service_is_running(container, service_name, service_owner,
service_proc)
# verify config file is not removed
run_container_cmd(container, f"test -f {env_path}")
def test_collector_package_install(distro):
pkg_name = "splunk-otel-collector"
pkg_dir = REPO_DIR / "dist"
service_name = "splunk-otel-collector"
service_owner = "splunk-otel-collector"
service_proc = "otelcol"
config_path = "/etc/otel/collector/splunk-otel-collector.conf"
bundle_dir = "/usr/lib/splunk-otel-collector/agent-bundle"
pkg_path = get_package(distro, pkg_name, pkg_dir)
assert pkg_path, f"{pkg_name} package not found in {pkg_dir}"
pkg_base = os.path.basename(pkg_path)
with run_distro_container(distro) as container:
copy_file_into_container(container, pkg_path, f"/test/{pkg_base}")
try:
# install package
if distro in DEB_DISTROS:
run_container_cmd(container, f"dpkg -i /test/{pkg_base}")
else:
run_container_cmd(container, f"rpm -i /test/{pkg_base}")
run_container_cmd(container, f"test -d {bundle_dir}")
run_container_cmd(container, f"test -d {bundle_dir}/run/collectd")
# verify service is not running after install without config file
time.sleep(5)
assert not service_is_running(container, service_name,
service_owner, service_proc)
# verify service starts with config file
run_container_cmd(container,
f"cp -f {config_path}.example {config_path}")
run_container_cmd(container, f"systemctl start {service_name}")
time.sleep(5)
assert wait_for(lambda: service_is_running(
container, service_name, service_owner, service_proc))
# verify service restart
run_container_cmd(container, f"systemctl restart {service_name}")
time.sleep(5)
assert wait_for(lambda: service_is_running(
container, service_name, service_owner, service_proc))
# verify service stop
run_container_cmd(container, f"systemctl stop {service_name}")
time.sleep(5)
assert not service_is_running(container, service_name,
service_owner, service_proc)
finally:
run_container_cmd(container,
f"journalctl -u {service_name} --no-pager")
# verify uninstall
run_container_cmd(container, f"systemctl start {service_name}")
time.sleep(5)
if distro in DEB_DISTROS:
run_container_cmd(container, f"dpkg -P {pkg_name}")
else:
run_container_cmd(container, f"rpm -e {pkg_name}")
time.sleep(5)
assert not service_is_running(container, service_name, service_owner,
service_proc)
# verify config file is not removed
run_container_cmd(container, f"test -f {config_path}")
def test_collector_package_install(distro):
pkg_path = get_package(distro, PKG_NAME, PKG_DIR)
assert pkg_path, f"{PKG_NAME} package not found in {PKG_DIR}"
pkg_base = os.path.basename(pkg_path)
with run_distro_container(distro) as container:
# install setcap dependency
if distro in RPM_DISTROS:
run_container_cmd(container, get_libcap_command(container))
else:
run_container_cmd(container, "apt-get update")
run_container_cmd(container, "apt-get install -y libcap2-bin")
copy_file_into_container(container, pkg_path, f"/test/{pkg_base}")
try:
# install package
if distro in DEB_DISTROS:
run_container_cmd(container, f"dpkg -i /test/{pkg_base}")
else:
run_container_cmd(container, f"rpm -i /test/{pkg_base}")
run_container_cmd(container, f"test -d {BUNDLE_DIR}")
run_container_cmd(container, f"test -d {BUNDLE_DIR}/run/collectd")
run_container_cmd(container, f"test -f {AGENT_CONFIG_PATH}")
run_container_cmd(container, f"test -f {GATEWAY_CONFIG_PATH}")
# verify service is not running after install without config file
time.sleep(5)
assert not service_is_running(container, SERVICE_NAME,
SERVICE_OWNER, SERVICE_PROC)
# verify service starts with config file
run_container_cmd(container,
f"cp -f {ENV_PATH}.example {ENV_PATH}")
run_container_cmd(container, f"systemctl start {SERVICE_NAME}")
time.sleep(5)
assert wait_for(lambda: service_is_running(
container, SERVICE_NAME, SERVICE_OWNER, SERVICE_PROC))
# verify service restart
run_container_cmd(container, f"systemctl restart {SERVICE_NAME}")
time.sleep(5)
assert wait_for(lambda: service_is_running(
container, SERVICE_NAME, SERVICE_OWNER, SERVICE_PROC))
# verify service stop
run_container_cmd(container, f"systemctl stop {SERVICE_NAME}")
time.sleep(5)
assert not service_is_running(container, SERVICE_NAME,
SERVICE_OWNER, SERVICE_PROC)
finally:
run_container_cmd(container,
f"journalctl -u {SERVICE_NAME} --no-pager")
# verify uninstall
run_container_cmd(container, f"systemctl start {SERVICE_NAME}")
time.sleep(5)
if distro in DEB_DISTROS:
run_container_cmd(container, f"dpkg -P {PKG_NAME}")
else:
run_container_cmd(container, f"rpm -e {PKG_NAME}")
time.sleep(5)
assert not service_is_running(container, SERVICE_NAME, SERVICE_OWNER,
SERVICE_PROC)
# verify config file is not removed
run_container_cmd(container, f"test -f {ENV_PATH}")
def test_installer_mode(distro, version, mode):
install_cmd = f"sh -x /test/install.sh -- testing123 --realm us0 --memory {TOTAL_MEMORY} --mode {mode}"
if version != "latest":
install_cmd = f"{install_cmd} --collector-version {version.lstrip('v')}"
if STAGE != "release":
assert STAGE in ("test", "beta"), f"Unsupported stage '{STAGE}'!"
install_cmd = f"{install_cmd} --{STAGE}"
print(f"Testing installation on {distro} from {STAGE} stage ...")
with run_distro_container(distro) as container:
# run installer script
copy_file_into_container(container, INSTALLER_PATH, "/test/install.sh")
try:
run_container_cmd(container,
install_cmd,
env={"VERIFY_ACCESS_TOKEN": "false"})
time.sleep(5)
config_path = AGENT_CONFIG_PATH if mode == "agent" else GATEWAY_CONFIG_PATH
if container.exec_run(f"test -f {OLD_CONFIG_PATH}").exit_code == 0:
config_path = OLD_CONFIG_PATH
elif mode == "gateway" and container.exec_run(
f"test -f {GATEWAY_CONFIG_PATH}").exit_code != 0:
config_path = AGENT_CONFIG_PATH
# verify env file created with configured parameters
splunk_env_path = SPLUNK_ENV_PATH
if container.exec_run(
f"test -f {OLD_SPLUNK_ENV_PATH}").exit_code == 0:
splunk_env_path = OLD_SPLUNK_ENV_PATH
run_container_cmd(
container,
f"grep '^SPLUNK_CONFIG={config_path}$' {splunk_env_path}")
run_container_cmd(
container,
f"grep '^SPLUNK_ACCESS_TOKEN=testing123$' {splunk_env_path}")
run_container_cmd(container,
f"grep '^SPLUNK_REALM=us0$' {splunk_env_path}")
run_container_cmd(
container,
f"grep '^SPLUNK_MEMORY_TOTAL_MIB={TOTAL_MEMORY}$' {splunk_env_path}"
)
# verify collector service status
assert wait_for(lambda: service_is_running(
container, service_owner=SERVICE_OWNER))
# the td-agent service should only be running when installing
# collector packages that have our custom fluent config
if container.exec_run(
"test -f /etc/otel/collector/fluentd/fluent.conf"
).exit_code == 0:
assert container.exec_run(
"systemctl status td-agent").exit_code == 0
else:
assert container.exec_run(
"systemctl status td-agent").exit_code != 0
# test support bundle script
# if container.exec_run("test -f /etc/otel/collector/splunk-support-bundle.sh").exit_code == 0:
# run_container_cmd(container, "/etc/otel/collector/splunk-support-bundle.sh -t /tmp/splunk-support-bundle")
# run_container_cmd(container, "test -f /tmp/splunk-support-bundle/config/agent_config.yaml")
# run_container_cmd(container, "test -f /tmp/splunk-support-bundle/logs/splunk-otel-collector.log")
# run_container_cmd(container, "test -f /tmp/splunk-support-bundle/logs/splunk-otel-collector.txt")
# if container.exec_run("test -f /etc/otel/collector/fluentd/fluent.conf").exit_code == 0:
# run_container_cmd(container, "test -f /tmp/splunk-support-bundle/logs/td-agent.log")
# run_container_cmd(container, "test -f /tmp/splunk-support-bundle/logs/td-agent.txt")
# run_container_cmd(container, "test -f /tmp/splunk-support-bundle/metrics/collector-metrics.txt")
# run_container_cmd(container, "test -f /tmp/splunk-support-bundle/metrics/df.txt")
# run_container_cmd(container, "test -f /tmp/splunk-support-bundle/metrics/free.txt")
# run_container_cmd(container, "test -f /tmp/splunk-support-bundle/metrics/top.txt")
# run_container_cmd(container, "test -f /tmp/splunk-support-bundle/zpages/tracez.html")
# run_container_cmd(container, "test -f /tmp/splunk-support-bundle.tar.gz")
run_container_cmd(container, "sh -x /test/install.sh --uninstall")
finally:
run_container_cmd(container, "journalctl -u td-agent --no-pager")
if container.exec_run(
"test -f /var/log/td-agent/td-agent.log").exit_code == 0:
run_container_cmd(container,
"cat /var/log/td-agent/td-agent.log")
run_container_cmd(container,
f"journalctl -u {SERVICE_NAME} --no-pager")
def test_installer_service_owner(distro, version):
service_owner = "test-user"
install_cmd = f"sh -x /test/install.sh -- testing123 --realm us0 --memory {TOTAL_MEMORY}"
install_cmd = f"{install_cmd} --service-user {service_owner} --service-group {service_owner}"
if version != "latest":
install_cmd = f"{install_cmd} --collector-version {version.lstrip('v')}"
if STAGE != "release":
assert STAGE in ("test", "beta"), f"Unsupported stage '{STAGE}'!"
install_cmd = f"{install_cmd} --{STAGE}"
print(f"Testing installation on {distro} from {STAGE} stage ...")
with run_distro_container(distro) as container:
copy_file_into_container(container, INSTALLER_PATH, "/test/install.sh")
try:
# run installer script
run_container_cmd(container,
install_cmd,
env={"VERIFY_ACCESS_TOKEN": "false"})
time.sleep(5)
config_path = AGENT_CONFIG_PATH
if container.exec_run(f"test -f {OLD_CONFIG_PATH}").exit_code == 0:
config_path = OLD_CONFIG_PATH
splunk_env_path = SPLUNK_ENV_PATH
if container.exec_run(
f"test -f {OLD_SPLUNK_ENV_PATH}").exit_code == 0:
splunk_env_path = OLD_SPLUNK_ENV_PATH
# verify env file created with configured parameters
run_container_cmd(
container,
f"grep '^SPLUNK_CONFIG={config_path}$' {splunk_env_path}")
run_container_cmd(
container,
f"grep '^SPLUNK_ACCESS_TOKEN=testing123$' {splunk_env_path}")
run_container_cmd(container,
f"grep '^SPLUNK_REALM=us0$' {splunk_env_path}")
run_container_cmd(
container,
f"grep '^SPLUNK_MEMORY_TOTAL_MIB={TOTAL_MEMORY}$' {splunk_env_path}"
)
# verify collector service status
assert wait_for(lambda: service_is_running(
container, service_owner=service_owner))
# the td-agent service should only be running when installing
# collector packages that have our custom fluent config
if container.exec_run(
"test -f /etc/otel/collector/fluentd/fluent.conf"
).exit_code == 0:
assert container.exec_run(
"systemctl status td-agent").exit_code == 0
else:
assert container.exec_run(
"systemctl status td-agent").exit_code != 0
finally:
run_container_cmd(container, "journalctl -u td-agent --no-pager")
run_container_cmd(container,
f"journalctl -u {SERVICE_NAME} --no-pager")