def _mock_python_binary_version(python_binary_name, version):
# type: (six.text_type, six.text_type) -> None
"""
Replace python binary with dummy srtipt that only print fake python version string.
:return:
"""
binary_path = BINARY_DIR_PATH / python_binary_name
binary_path_backup_path = Path(six.text_type(binary_path) + "_bc")
# this function was used previously delete old backup.
if binary_path_backup_path.exists():
shutil.copy(
six.text_type(binary_path_backup_path),
six.text_type(binary_path),
)
os.remove(six.text_type(binary_path_backup_path))
if not version:
os.system("python2 --version")
return
if not binary_path.exists():
return
# make backup of the original binary in case if we want to keep using it.
shutil.copy(six.text_type(binary_path),
six.text_type(binary_path_backup_path))
os.remove(six.text_type(binary_path))
# write new source to binary file. Now it just returns fake version.
with binary_path.open("w") as f:
f.write("#!/bin/bash\n")
f.write("echo Python {0}\n".format(version))
os.chmod(six.text_type(binary_path), stat.S_IWRITE | stat.S_IEXEC)
class CentOSBuilderBase(AgentImageBuilder):
IMAGE_TAG = create_distribution_base_image_name(DISTRIBUTION_NAME)
DOCKERFILE = Path(__file__).parent / "Dockerfile.base"
INCLUDE_PATHS = [
Path(get_install_root(), "dev-requirements.txt"),
Path(get_install_root(), "py26-unit-tests-requirements.txt"),
]
def test_config(request, agent_env_settings_fields):
"""
load config file as dict if it is located in project root and its path specified in pytest command line.
If it is not specified, return empty dict.
"""
config_path = request.config.getoption("--test-config")
if config_path and Path(config_path).exists():
config_path = Path(config_path)
with config_path.open("r") as f:
config = yaml.safe_load(f)
else:
config = dict()
return config
class CentOSBuilder(AgentImageBuilder):
IMAGE_TAG = create_distribution_image_name(DISTRIBUTION_NAME)
DOCKERFILE = Path(__file__).parent / "Dockerfile"
REQUIRED_IMAGES = [FpmPackageBuilder, CentOSBuilderBase]
REQUIRED_CHECKSUM_IMAGES = [CentOSBuilderBase]
COPY_AGENT_SOURCE = True
IGNORE_CACHING = True
def pytest_addoption(parser):
parser.addoption(
"--test-config",
action="store",
default=six.text_type(Path(__file__).parent / "config.yml"),
help=
"Path to yaml file with essential agent settings and another test related settings. "
"Fields from this config file will be set as environment variables.",
)
parser.addoption(
"--no-dockerize",
action="store_true",
help=
"Make test cases that were decorated by 'utils.dockerized_case' run on that machine, "
"not inside docker container. "
"Also used by 'utils.dockerized_case' when test case is already in container "
"to run actual test case and to prevent another container creation.",
)
parser.addoption(
"--no-rebuild",
action="store_true",
help="Build only final image and do not build required base images.",
)
parser.addoption(
"--artifacts-path",
help=
"Path to directory where tests cases can store their results and artifacts.",
)
def worker_sessions_log_paths(self):
"""Get list of log file path for all worker sessions."""
result = []
for worker_session_id in self.config_object.get_session_ids_from_all_workers(
):
log_file_path = self.config_object.get_worker_session_agent_log_path(
worker_session_id)
result.append(Path(log_file_path))
return result
def _agent_config(self):
# type: () -> Dict[six.text_type, Any]
"""
Build and return agent configuration.
:return: dict with configuration.
"""
# do not include default log files.
files_to_exclude_from_config = [
str(Path(self.agent_logs_dir_path, name)) # type:ignore
for name in [
"linux_process_metrics.log",
"linux_system_metrics.log",
"agent.log",
]
]
config_log_files = list()
for log_file in self._log_files.values():
if log_file["path"] not in files_to_exclude_from_config:
config_log_files.append(log_file)
config = {
"api_key": compat.os_environ_unicode["SCALYR_API_KEY"],
"verify_server_certificate": "false",
"server_attributes": {
"serverHost": self._server_host
},
"logs": config_log_files,
"default_sessions_per_worker": self._worker_sessions_count,
"monitors": [],
"use_multiprocess_workers": self._workers_type == "process",
# NOTE: We disable this functionality so tests finish faster and we can use lower
# timeout
"global_monitor_sample_interval_enable_jitter": False,
}
if self._enable_debug_log:
# NOTE: We also enable copy_from_start if debug_level is enabled to we ship whole debug
# log to scalyr
config["debug_level"] = 5
config["logs"].append({"path": "agent_debug.log"}) # type: ignore
if not self._send_to_server:
# do not send requests to server.
config["disable_send_requests"] = True
# Print out the agent config (masking the secrets) to make troubleshooting easier
config_sanitized = copy.copy(config)
config_sanitized.pop("api_key", None)
print("Using agent config: %s" % (pprint.pformat(config_sanitized)))
return config
def handle_command_line(cls):
parser = argparse.ArgumentParser()
parser.add_argument(
"--dockerfile",
action="store_true",
help="Print dockerfile content of the image.",
)
parser.add_argument(
"--checksum",
action="store_true",
help=
"Print base64 encoded sha256 checksum of the Dockerfile of this builder. "
"Also, it counts checksum of all required builders.",
)
parser.add_argument(
"--name",
action="store_true",
help="Get name of the image which is built by this builder.",
)
parser.add_argument(
"--build-with-cache",
type=six.text_type,
help=
"Path to cache directory. If specified, firstly, the builder searches for serialized tar file of the image,"
"If this file does not exist, builds it from scratch and saves there.",
)
args = parser.parse_args()
if args.checksum:
checksum_object = cls.get_checksum()
base64_checksum = checksum_object.hexdigest()
print(base64_checksum)
exit(0)
if args.name:
print(cls.IMAGE_TAG)
exit(0)
if args.build_with_cache:
builder = cls()
builder.build_with_cache(Path(args.build_with_cache),
skip_if_exists=False)
exit(0)
def _get_default_paths(self): # type: () -> Dict[six.text_type, Path]
"""
Get default path for essential directories and files of the agent. Those paths are fetched from 'PlatformController'.
"""
# create new 'PlatformController' instance. Since this code is executed on the same machine with agent,
# platform setting and paths should match.
platform = PlatformController.new_platform()
# change install type of the controller to needed one.
platform._install_type = self._installation_type
default_types = platform.default_paths
result = dict()
for k, v in default_types.__dict__.items():
result[k] = Path(v)
return result
class CommonMonitorBuilder(AgentImageBuilder):
IMAGE_TAG = "scalyr-agent-testings-monitor-common"
DOCKERFILE = Path(__file__).parent / "Dockerfile"
INCLUDE_PATHS = [
Path(__file__).parent / "init.sql",
Path(Path(__file__).parent / "nginx-config"),
Path(Path(__file__).parent, "dummy-flask-server.py"),
]
REQUIRED_IMAGES = [BaseMonitorBuilder]
REQUIRED_CHECKSUM_IMAGES = [BaseMonitorBuilder]
COPY_AGENT_SOURCE = True
IGNORE_CACHING = True
def __init__(self):
self._docker = None # type: Optional
# dict with files which need to be copied to build_context.
# New paths can be added by using 'add_to_build_context' method.
self._things_copy_to_build_context = dict() # type: Dict[Path, Dict]
# copy agent course code if needed.
if type(self).COPY_AGENT_SOURCE:
root_path = Path(get_package_root()).parent
self.add_to_build_context(root_path,
"agent_source",
custom_copy_function=_copy_agent_source)
# the value of this attribute is the path to the file to be copied to the image build
# context.
for path in self.INCLUDE_PATHS:
self.add_to_build_context(path, path.name)
def build(self, image_cache_path=None, skip_requirements=False):
"""
Build docker image.
:param image_cache_path: import image from .tar files located in this directory, if exist.
:param skip_requirements: Build only image for this builder and skip all required builders.
"""
# if image caching is enabled and image exists we assume that image has already built in previous test cases.
if image_cache_path is not None:
if self.is_image_exists():
print("Image '{0}' already exists. Skip build.".format(
self.IMAGE_TAG))
return
if not skip_requirements:
# build all required images.
for required_image_builder_cls in type(self).REQUIRED_IMAGES:
builder = required_image_builder_cls()
builder.build(image_cache_path=image_cache_path)
if not type(self).IGNORE_CACHING and image_cache_path is not None:
self.build_with_cache(Path(image_cache_path))
return
print("Build image: '{0}'".format(self.image_tag))
build_context_path = create_tmp_directory(
suffix="{0}-build-context".format(self.image_tag))
dockerfile_path = build_context_path / "Dockerfile"
dockerfile_path.write_text(self.get_dockerfile_content())
self._copy_to_build_context(build_context_path)
_, output_gen = self._docker_client.images.build(
tag=self.image_tag,
path=six.text_type(build_context_path),
dockerfile=six.text_type(dockerfile_path),
rm=True,
)
shutil.rmtree(six.text_type(build_context_path), ignore_errors=True)
for chunk in output_gen:
print(chunk.get("stream", ""), end="")
def _get_current_config_script_name():
return Path(
os.readlink(
six.text_type(SCALYR_PACKAGE_BIN_PATH /
"scalyr-agent-2-config"))).name
def wrapper(self, path, *args, **kwargs):
if isinstance(path, six.text_type):
path = Path(path)
return fn(self, path, *args, **kwargs)
class BaseMonitorBuilder(AgentImageBuilder):
IMAGE_TAG = "scalyr-agent-testings-monitor-base"
DOCKERFILE = Path(__file__).parent / "Dockerfile"
INCLUDE_PATHS = [
Path(get_install_root(), "dev-requirements.txt"),
]
import json
import pprint
import subprocess
from distutils.spawn import find_executable
from scalyr_agent.__scalyr__ import PACKAGE_INSTALL, DEV_INSTALL, get_package_root
from scalyr_agent import compat
from scalyr_agent.platform_controller import PlatformController
from tests.utils.compat import Path
from tests.utils.common import get_env
import six
_AGENT_MAIN_PATH = Path(get_package_root(), "agent_main.py")
_CONFIG_MAIN_PATH = Path(get_package_root(), "config_main.py")
def _make_or_clear_directory(path): # type: (Path) -> None
"""
Create directory or clear it if exests..
"""
if path.exists():
shutil.rmtree(six.text_type(path), ignore_errors=True)
path.mkdir(exist_ok=True, parents=True)
def _path_or_text(fn):
def wrapper(self, path, *args, **kwargs):
if isinstance(path, six.text_type):
class FpmPackageBuilder(AgentImageBuilder):
IMAGE_TAG = "scalyr-agent-testings-fpm_package-builder"
DOCKERFILE = Path(__file__).parent / "Dockerfile"
def create_tmp_file(suffix=""):
# type: (six.text_type) -> Path
tmp_file = tempfile.NamedTemporaryFile(prefix=TEMP_PREFIX, suffix="-" + suffix)
tmp_file.close()
return Path(tmp_file.name)
def wrapper(request, *args, **kwargs):
no_dockerize = request.config.getoption("--no-dockerize")
if no_dockerize:
result = f(request, *args, **kwargs)
return result
builder = builder_cls()
no_rebuild = request.config.getoption("--no-rebuild", False)
if builder.is_image_exists():
# we rebuild image if there is no option to skip rebuild.
if not no_rebuild:
builder.build(skip_requirements=True)
else:
try:
builder.build(skip_requirements=no_rebuild)
except docker.errors.BuildError as e:
# Throw a more user-friendly exception if the base image doesn't exist
if "does not exist" in str(e) and "-base" in str(e):
try:
base_image_name = builder.REQUIRED_CHECKSUM_IMAGES[
0].IMAGE_TAG
except Exception:
base_image_name = "unknown"
msg = (
'Base container image "%s" doesn\'t exist and --no-rebuild flag is '
"used. You need to either manually build the base image or remove "
"the --no-rebuild flag.\n\nOriginal error: %s" %
(base_image_name, str(e)))
raise Exception(msg)
docker_client = docker.from_env()
container_name = "{0}-{1}-{2}".format(
builder.image_tag,
Path(file_path).name.replace(".py", ""), func_name)
try:
# remove container if it was created previously.
container = docker_client.containers.get(container_name)
container.remove()
except docker.errors.NotFound:
pass
print("Create container '{0}' from '{1}' image.".format(
container_name, builder.image_tag))
container = docker_client.containers.run(
builder.image_tag,
name=container_name,
detach=True,
command=command,
stdout=True,
stderr=True,
environment=get_environment_for_docker_run(),
)
exit_code = container.wait()["StatusCode"]
logs = six.ensure_text(container.logs(follow=True))
print(logs)
# save logs if artifacts path is specified.
artifacts_path = request.config.getoption("--artifacts-path", None)
if artifacts_path:
coverage_file_path = Path("/", ".coverage")
artifacts_path = Path(artifacts_path)
if artifacts_use_subdirectory:
# We run each test case in a new container instance so we make sure we store
# logs under a sub-directory which matches the test function name
artifacts_path = artifacts_path / func_name
file_paths_to_copy.add(six.text_type(coverage_file_path))
copy_artifacts(
container=container,
file_paths=file_paths_to_copy,
destination_path=artifacts_path,
)
if remove_container:
container.remove(force=True)
print("Container '{0}' removed.".format(builder.image_tag))
# raise failed assertion, due to non-zero result from container.
if exit_code:
raise AssertionError(
"Test case inside container failed (container exited with %s "
"status code)." % (exit_code))
import glob
import stat
from io import open
import six
import pytest
from scalyr_agent import compat
from tests.utils.compat import Path
from tests.utils.common import get_shebang_from_file
from tests.utils.agent_runner import AgentRunner, PACKAGE_INSTALL
from tests.common import PackageInstallationError
from tests.common import install_deb, remove_deb
SCALYR_PACKAGE_BIN_PATH = Path("/", "usr", "share", "scalyr-agent-2", "bin")
# NOTE: Binary dir path is different across distros that's why we use which to locate it
BINARY_DIR_PATH = Path("/", "usr", "bin")
def _get_python_major_version(runner):
status = json.loads(runner.status_json())
version_string = status["python_version"]
version = int(version_string[0])
return version
def create_tmp_directory(suffix=""):
# type: (six.text_type) -> Path
path = Path(tempfile.mkdtemp(prefix=TEMP_PREFIX, suffix="-" + suffix))
return path
