def test_cookiecutter_presence(
container: docker.models.containers.Container) -> None:
"""Is cookiecutter installed?"""
res: docker.ExecResult = container.exec_run("cookiecutter --version")
assert (res.exit_code == 0
), f"cookiecutter not correctly or not installed in container"
def run_oozie_examples_with_dbd(
oozieserver: docker.models.containers.Container, logfile: str,
report_file: str, whitelist: List[str], blacklist: List[str],
validate: List[str], timeout: int) -> int:
"""
Runs the Oozie examples in the Oozie server docker container.
Args:
oozieserver: The object representing the Oozie server.
logfile: The path on the container's file system to the logfile that will
be written by the script that runs the examples within the container.
report_file: The path on the container's file system to the report file that will
be written by the script that runs the examples within the container.
whitelist: A list of examples that should be run. If provided, only these examples
will be run, otherwise all non-blacklisted examples will be run.
blacklist: A list of examples that should not be run.
validate: A list of fluent examples that should only be validated, not run.
timeout: The timeout after which running examples are killed.
Returns:
The exit code of the process running the test, which is 1 if any tests failed.
"""
cmd_base = "python3 /opt/oozie/inside_container/example_runner.py --logfile {} --report {}".format(
logfile, report_file)
cmd_w_b_list = _command_with_whitelist_and_blacklist(
cmd_base, whitelist, blacklist, validate)
cmd = cmd_w_b_list + " -t {}".format(timeout)
logging.info("Running the Oozie examples with command %s.", cmd)
(errcode, _) = oozieserver.exec_run(cmd, workdir="/opt/oozie")
logging.info("Testing finished with exit code %s.", errcode)
return errcode
def _container_exec(container: docker.models.containers.Container,
cmd: str) -> str:
exit_code, value = container.exec_run(cmd)
value = value.decode('utf-8').strip()
if exit_code:
logger.error("Error. CMD: {} ; Output: {}".format(cmd, value))
return ''
return value
def test_is_role_installed(container: docker.models.containers.Container,
installation_dir) -> None:
"""Is role installed?"""
res: docker.ExecResult = container.exec_run(
f"cookiecutter --no-input gh:fesaille/cookiecutter-ansible-role -o {installation_dir}"
)
assert res.exit_code == 0, f"Role not correctly or not installed in container"
def _run_container_setup(container: docker.models.containers.Container,
target_adapter: Type['BaseTargetAdapter']) -> None:
logger.info('Running initialization commands in container...')
for command in target_adapter.image_initialize_bash_commands():
response = container.exec_run(f"/bin/bash -c '{command}'", tty=True)
if response[0] > 0:
raise OSError(response[1])
logger.info('Setup commands finished.')
def test_is_git_dir(container: docker.models.containers.Container,
installation_dir: Path, config: Dict) -> None:
"""Is role inits a git dir?"""
role_name = config["role_name"]
role_dir = installation_dir / role_name
res: docker.ExecResult = container.exec_run(f"git -C {role_dir} rev-parse")
assert res.exit_code == 0, f"Role did not create a git dir"
def _run_command(
self,
container: docker.models.containers.Container,
command: List[str],
stdout: bool,
stderr: bool,
raise_on_failure: Optional[bool] = True) -> Tuple[int, str]:
"""Run docker commands using docker python sdk.
Args:
container: The docker container that runs this command.
command: The command to run in docker container.
stdout: Whether to include stdout in output.
stderr: Whether to include stderr in output.
raise_on_failure: Whether to raise on failure.
Returns:
A tuple containing the return code of the given command and the
output of that command.
"""
try:
returncode, output = container.exec_run(command,
stdout=stdout,
stderr=stderr)
output = output.decode('utf-8')
except docker.errors.APIError as e:
# Clean up the container if command fails
self._cleanup_container(container)
raise PipCheckerError(error_msg="Error occurs when executing "
"commands in container."
"Error message: "
"{}".format(e.explanation))
except IOError as e:
# TODO: Log the exception and monitor it after trying to decode
# this into a requests.exception.* e.g. ReadTimeout. See:
# http://docs.python-requests.org/en/master/_modules/requests/exceptions/
raise PipCheckerError(
error_msg="An error occurred while running the command {} in"
"container. Error message: {}".format(command, e))
# Checking for cases where the command was killed by a signal.
# If a process was killed by a signal, then it's exit code will be
# 128 + <signal number>.
# If a docker container exits with a running command then it will be
# killed with SIGKILL => 128 + 9 = 137
if returncode > 128 and returncode <= 137:
raise PipCheckerError(
error_msg="The command {} was killed by signal {}. "
"This likely means that the Docker container timed "
"out. Error msg: {}".format(command, returncode - 128, output))
elif returncode and raise_on_failure:
raise PipError(error_msg=output,
command=command,
returncode=returncode)
return returncode, output
def _remount_replica_data(
container: docker.models.containers.Container,
target_adapter: Type['BaseTargetAdapter']) -> None:
logger.info('Remounting data inside target...')
for command in target_adapter.image_finalize_bash_commands():
response = container.exec_run(f"/bin/bash -c '{command}'",
tty=True)
if response[0] > 0:
raise OSError(response[1])
logger.info('Data remounted, image ready to be finalized.')
def destroy(container: docker.models.containers.Container) -> None:
"""
Destroy a container containing a block device that is used with
loopback devices. Also removes the loopback device.
Args:
container: The container to destroy.
"""
exit_code, output = container.exec_run(
cmd=['cat', 'loopback_device_path'], )
assert exit_code == 0, output.decode()
path = output.decode().rstrip()
exit_code, output = container.exec_run(cmd=['losetup', '-d', path])
assert exit_code == 0, output.decode()
container.stop()
container.remove(v=True)
def test_python_installation(
container: docker.models.containers.Container) -> None:
"""Is python available in the container?"""
python_version_tupled = tuple(
int(p) for p in str(PYTHON_VERSION).split("."))
res: docker.ExecResult = container.exec_run(
f'python -c "import sys; assert sys.version_info >= {python_version_tupled}"'
)
assert res.exit_code == 0, f"Python version in container not {PYTHON_VERSION}"
def upload_examples_to_hdfs(
oozieserver: docker.models.containers.Container) -> None:
"""
Uploads the Oozie examples from the Oozie server's local file system to HDFS.
Args:
oozieserver: The object representing the Oozie server.
"""
logging.info("Uploading the tests to hdfs.")
(errcode, _) = oozieserver.exec_run(
"/bin/bash /opt/oozie/inside_container/prepare_examples.sh",
workdir="/opt/oozie")
logging.info("Uploading the tests finished with exit code: %s.", errcode)
def _run_command(self,
container: docker.models.containers.Container,
command: List[str],
stdout: bool,
stderr: bool,
raise_on_failure: Optional[bool] = True) -> (int, str):
"""Run docker commands using docker python sdk.
Args:
container: The docker container that runs this command.
command: The command to run in docker container.
stdout: Whether to include stdout in output.
stderr: Whether to include stderr in output.
raise_on_failure: Whether to raise on failure.
Returns:
A tuple containing the return code of the given command and the
output of that command.
"""
try:
returncode, output = container.exec_run(command,
stdout=stdout,
stderr=stderr)
output = output.decode('utf-8')
except docker.errors.APIError as e:
# Clean up the container if command fails
self._cleanup_container(container)
raise PipCheckerError(error_msg="Error occurs when executing "
"commands in container."
"Error message: "
"{}".format(e.explanation))
if returncode and raise_on_failure:
raise PipError(error_msg=output,
command=command,
returncode=returncode)
return returncode, output
def _run_cmd(
ctr: docker.models.containers.Container,
cmd: Union[str, List[str]],
) -> Tuple[int, str]:
retcode, output = ctr.exec_run(cmd)
return retcode, output.decode("utf8").strip()
def test_pre_commit_presence(
container: docker.models.containers.Container) -> None:
"""Is pre-commit installed?"""
res: docker.ExecResult = container.exec_run("pre-commit --version")
assert res.exit_code == 0, f"pre-commit not correctly or not installed in container"
def exec_run(container: docker.models.containers.Container, command: str, **kwargs) -> typing.Callable[[], docker.models.containers.ExecResult]:
"""
Invoke the given command within the given container.
Returns a callback object
"""
return container.exec_run(command, **kwargs)