def wait_for_container_exit( self, container: docker.models.containers.Container) -> dict: logger.info('(%s) container launched, waiting for exit...', container.id) for entry in container.logs(timestamps=True, follow=True): logger.info("(%s) %s", container.id, entry) return container.wait(timeout=self.__docker_wait_timeout)
def _cleanup_container(self, container: docker.models.containers.Container): """Stop the container and remove it's associated storage.""" try: container.stop(timeout=0) except (docker.errors.APIError, docker.errors.NotFound): raise PipCheckerError( error_msg="Error occurs when cleaning up docker container." "Container does not exist.")
def _cleanup_container(self, container: docker.models.containers.Container): """Remove the container.""" try: container.stop() container.remove() except (docker.errors.APIError, docker.errors.NotFound): raise PipCheckerError( error_msg="Error occurs when cleaning up docker container." "Container does not exist.")
def run_oozie_examples_with_dbd( oozieserver: docker.models.containers.Container, logfile: str, report_file: str, whitelist: List[str], blacklist: List[str], validate: List[str], timeout: int) -> int: """ Runs the Oozie examples in the Oozie server docker container. Args: oozieserver: The object representing the Oozie server. logfile: The path on the container's file system to the logfile that will be written by the script that runs the examples within the container. report_file: The path on the container's file system to the report file that will be written by the script that runs the examples within the container. whitelist: A list of examples that should be run. If provided, only these examples will be run, otherwise all non-blacklisted examples will be run. blacklist: A list of examples that should not be run. validate: A list of fluent examples that should only be validated, not run. timeout: The timeout after which running examples are killed. Returns: The exit code of the process running the test, which is 1 if any tests failed. """ cmd_base = "python3 /opt/oozie/inside_container/example_runner.py --logfile {} --report {}".format( logfile, report_file) cmd_w_b_list = _command_with_whitelist_and_blacklist( cmd_base, whitelist, blacklist, validate) cmd = cmd_w_b_list + " -t {}".format(timeout) logging.info("Running the Oozie examples with command %s.", cmd) (errcode, _) = oozieserver.exec_run(cmd, workdir="/opt/oozie") logging.info("Testing finished with exit code %s.", errcode) return errcode
def test_cookiecutter_presence( container: docker.models.containers.Container) -> None: """Is cookiecutter installed?""" res: docker.ExecResult = container.exec_run("cookiecutter --version") assert (res.exit_code == 0 ), f"cookiecutter not correctly or not installed in container"
def get_file(container: docker.models.containers.Container, path: str, dest: str = ".") -> None: """Get a single file from a container. :param path: The path to retrieve. :param dest: The filename to extract to. If it is an existing directory, the filename from the tar file will be used. """ if os.path.isdir(dest): fname = None else: fname = dest it, _ = container.get_archive(path) reader = _DLReader(it) tf = tarfile.TarFile(mode="r", fileobj=reader) for member in tf: ## We only need the first one f = tf.extractfile(member) if fname is None: fname = os.path.join(dest, member.name) with open(fname, "wb") as of: shutil.copyfileobj(f, of)
def _get_ffmpeg_process_data( container: docker.models.containers.Container) -> (str, str): """ returned pid, cpu usage """ top = [] empty_result = '', '' if container.status != 'running': return empty_result try: # [ (pid, cmd), (pid, cmd), .. ] top = container.top(ps_args='-eo pid,comm,pcpu') except docker.errors.APIError: logger.error("Error while trying get container top.\n" + traceback.format_exc()) return empty_result if not top or 'Processes' not in top: logger.warning( f"Warning. Top is empty or format is wrong. container:'{container.name}', top: {top}" ) return empty_result try: top = top['Processes'] for pid, cmd, pcpu in top: if 'ffmpeg' in cmd: return pid, pcpu return empty_result except ValueError: logger.warning( f"Warning. Format of top is wrong.container:'{container.name}', top: {top}" ) logger.warning(traceback.format_exc()) return empty_result
def _cleanup_container(self, container: docker.models.containers.Container): """Stop the container and remove it's associated storage.""" try: container.stop(timeout=0) except (docker.errors.APIError, docker.errors.NotFound): raise PipCheckerError( error_msg="Error occurs when cleaning up docker container." "Container does not exist.") except IOError as e: # TODO: Log the exception and monitor it after trying to decode # this into a requests.exception.* e.g. ReadTimeout. See: # http://docs.python-requests.org/en/master/_modules/requests/exceptions/ raise PipCheckerError( error_msg="An error occurred while stopping a docker" "container. Error message: {}".format(e))
def get_stats(container: docker.models.containers.Container) -> dict: stats = container.stats(stream=False) stats.update({'attrs': container.attrs}) # print(stats) filtered = _filter_stats(stats) # print(filtered) return filtered
def test_is_role_installed(container: docker.models.containers.Container, installation_dir) -> None: """Is role installed?""" res: docker.ExecResult = container.exec_run( f"cookiecutter --no-input gh:fesaille/cookiecutter-ansible-role -o {installation_dir}" ) assert res.exit_code == 0, f"Role not correctly or not installed in container"
def _container_exec(container: docker.models.containers.Container, cmd: str) -> str: exit_code, value = container.exec_run(cmd) value = value.decode('utf-8').strip() if exit_code: logger.error("Error. CMD: {} ; Output: {}".format(cmd, value)) return '' return value
def _run_container_setup(container: docker.models.containers.Container, target_adapter: Type['BaseTargetAdapter']) -> None: logger.info('Running initialization commands in container...') for command in target_adapter.image_initialize_bash_commands(): response = container.exec_run(f"/bin/bash -c '{command}'", tty=True) if response[0] > 0: raise OSError(response[1]) logger.info('Setup commands finished.')
def test_is_git_dir(container: docker.models.containers.Container, installation_dir: Path, config: Dict) -> None: """Is role inits a git dir?""" role_name = config["role_name"] role_dir = installation_dir / role_name res: docker.ExecResult = container.exec_run(f"git -C {role_dir} rev-parse") assert res.exit_code == 0, f"Role did not create a git dir"
def _run_command( self, container: docker.models.containers.Container, command: List[str], stdout: bool, stderr: bool, raise_on_failure: Optional[bool] = True) -> Tuple[int, str]: """Run docker commands using docker python sdk. Args: container: The docker container that runs this command. command: The command to run in docker container. stdout: Whether to include stdout in output. stderr: Whether to include stderr in output. raise_on_failure: Whether to raise on failure. Returns: A tuple containing the return code of the given command and the output of that command. """ try: returncode, output = container.exec_run(command, stdout=stdout, stderr=stderr) output = output.decode('utf-8') except docker.errors.APIError as e: # Clean up the container if command fails self._cleanup_container(container) raise PipCheckerError(error_msg="Error occurs when executing " "commands in container." "Error message: " "{}".format(e.explanation)) except IOError as e: # TODO: Log the exception and monitor it after trying to decode # this into a requests.exception.* e.g. ReadTimeout. See: # http://docs.python-requests.org/en/master/_modules/requests/exceptions/ raise PipCheckerError( error_msg="An error occurred while running the command {} in" "container. Error message: {}".format(command, e)) # Checking for cases where the command was killed by a signal. # If a process was killed by a signal, then it's exit code will be # 128 + <signal number>. # If a docker container exits with a running command then it will be # killed with SIGKILL => 128 + 9 = 137 if returncode > 128 and returncode <= 137: raise PipCheckerError( error_msg="The command {} was killed by signal {}. " "This likely means that the Docker container timed " "out. Error msg: {}".format(command, returncode - 128, output)) elif returncode and raise_on_failure: raise PipError(error_msg=output, command=command, returncode=returncode) return returncode, output
def copy_files_container(container: docker.models.containers.Container, files: FILES_SRC_TARGET): """ Args: container: the container object. files: a list of (target path in container, source path in machine). """ if files: with tempfile.NamedTemporaryFile() as tar_file_path: with tarfile.open(name=tar_file_path.name, mode='w') as tar_file: for src, dst in files: try: tar_file.add(src, arcname=dst) except Exception as error: logger.debug(error) with open(tar_file_path.name, 'rb') as byte_file: container.put_archive('/', byte_file.read())
def put_file( container: docker.models.containers.Container, path: str, fname: str, name: str = None, mode: int = 0o640, ) -> None: """Put a single file into a container. Only works on single regular files and ignores all metadata. :param path: The directory in the container to extract to. :param fname: The filename of the local file. :param name: The name to store in the tar file; defaults to the basename of the file. :param mode: The mode for the stored file (3-digit octal number). """ with _SingleFileTar(fname, name, mode=mode) as f: container.put_archive(path, f)
def _remount_replica_data( container: docker.models.containers.Container, target_adapter: Type['BaseTargetAdapter']) -> None: logger.info('Remounting data inside target...') for command in target_adapter.image_finalize_bash_commands(): response = container.exec_run(f"/bin/bash -c '{command}'", tty=True) if response[0] > 0: raise OSError(response[1]) logger.info('Data remounted, image ready to be finalized.')
def test_python_installation( container: docker.models.containers.Container) -> None: """Is python available in the container?""" python_version_tupled = tuple( int(p) for p in str(PYTHON_VERSION).split(".")) res: docker.ExecResult = container.exec_run( f'python -c "import sys; assert sys.version_info >= {python_version_tupled}"' ) assert res.exit_code == 0, f"Python version in container not {PYTHON_VERSION}"
def upload_examples_to_hdfs( oozieserver: docker.models.containers.Container) -> None: """ Uploads the Oozie examples from the Oozie server's local file system to HDFS. Args: oozieserver: The object representing the Oozie server. """ logging.info("Uploading the tests to hdfs.") (errcode, _) = oozieserver.exec_run( "/bin/bash /opt/oozie/inside_container/prepare_examples.sh", workdir="/opt/oozie") logging.info("Uploading the tests finished with exit code: %s.", errcode)
def copy_host_to_container(container: docker.models.containers.Container, file: str, dest_path: str) -> None: """Copies a file or a folder from the host to the container. file may be a source folder or a file.""" if Path(file).is_dir(): archive = create_archive(file, arcname=Path(dest_path).name) container.put_archive(path=Path(dest_path).parent.as_posix(), data=archive) else: archive = create_archive(file) container.put_archive(path=dest_path, data=archive) archive.close() with create_archive(file) as archive: container.put_archive(path=dest_path, data=archive)
def convert_container_to_replica( self, replica_name: str, container: docker.models.containers.Container) -> docker.models.images.Image: """coerces a live container into a replica image and returns the image. replica_name: the name of the new replica """ replica_name = self.sanitize_replica_name(replica_name) logger.info(f'Creating new replica image with name {replica_name}...') try: self.client.images.remove(replica_name, force=True) except docker.errors.ImageNotFound: pass replica = container.commit( repository=self.sanitize_replica_name(replica_name) ) logger.info(f'Replica image {replica.tags[0]} created. Cleaning up...') self.remove_container(container.name) return replica
def _collect_container_data( container: docker.models.containers.Container) -> dict: data = {} data['id'] = container.id data['name'] = container.name if len(container.image.tags) > 0: data['image'] = container.image.tags[0] data['started_at'] = container.attrs['State']['StartedAt'] data['finished_at'] = container.attrs['State']['FinishedAt'] data['finished_at_formatted'] = data['finished_at'].replace('T', ' ') data['finished_at_formatted'] = ' '.join( data['finished_at_formatted'].split('.')[:-1]) data['cmd'] = ' '.join(container.attrs['Args']) data['base_pid'] = container.attrs['State']['Pid'] pid, pcpu = _get_ffmpeg_process_data(container) data['ffmpeg_pid'] = pid data['cpu_usage'] = pcpu if container.status == "running": now = datetime.datetime.now(pytz.timezone("UTC")).replace(tzinfo=None) started = _parse_docker_dt(data['started_at']) up = now - started hours = up.seconds // 3600 minutes = (up.seconds % 3600) // 60 seconds = up.seconds % 60 data['up'] = '{}d:{}h:{}m:{}s'.format(up.days, hours, minutes, seconds) data['status'] = container.status if container.status == "running": try: data['last_log'] = container.logs(tail=LAST_DOCKER_LOGS_COUNT, timestamps=True) data['last_log'] = data['last_log'].decode('utf-8') except docker.errors.APIError: logger.error(traceback.format_exc()) data['restart_count'] = container.attrs['RestartCount'] data['created'] = container.attrs['Created'] data['is_host_network'] = False networks = container.attrs['NetworkSettings'].get('Networks') if networks and 'host' in networks: data['is_host_network'] = True return data
def _run_command(self, container: docker.models.containers.Container, command: List[str], stdout: bool, stderr: bool, raise_on_failure: Optional[bool] = True) -> (int, str): """Run docker commands using docker python sdk. Args: container: The docker container that runs this command. command: The command to run in docker container. stdout: Whether to include stdout in output. stderr: Whether to include stderr in output. raise_on_failure: Whether to raise on failure. Returns: A tuple containing the return code of the given command and the output of that command. """ try: returncode, output = container.exec_run(command, stdout=stdout, stderr=stderr) output = output.decode('utf-8') except docker.errors.APIError as e: # Clean up the container if command fails self._cleanup_container(container) raise PipCheckerError(error_msg="Error occurs when executing " "commands in container." "Error message: " "{}".format(e.explanation)) if returncode and raise_on_failure: raise PipError(error_msg=output, command=command, returncode=returncode) return returncode, output
def copy_container_to_host(container: docker.models.containers.Container, file: str, dest_path: str, maxsize: int = 0) -> None: """ Copies a file from a container to the host. dest_path needs to be the destination directory. Max size may be limited by maxsize. Value of 0 means there's no limitation. """ stream = io.BytesIO() bits, stat = container.get_archive(file) if maxsize > 0 and stat['size'] > maxsize: raise FileTooBigException(size=stat['size'], max_size=maxsize, filename=stat['name']) for chunk in bits: stream.write(chunk) stream.seek(0) with tarfile.TarFile(fileobj=stream) as archive: archive.extractall(dest_path)
def destroy(container: docker.models.containers.Container) -> None: """ Destroy a container containing a block device that is used with loopback devices. Also removes the loopback device. Args: container: The container to destroy. """ exit_code, output = container.exec_run( cmd=['cat', 'loopback_device_path'], ) assert exit_code == 0, output.decode() path = output.decode().rstrip() exit_code, output = container.exec_run(cmd=['losetup', '-d', path]) assert exit_code == 0, output.decode() container.stop() container.remove(v=True)
def exec_run(container: docker.models.containers.Container, command: str, **kwargs) -> typing.Callable[[], docker.models.containers.ExecResult]: """ Invoke the given command within the given container. Returns a callback object """ return container.exec_run(command, **kwargs)
def _run_cmd( ctr: docker.models.containers.Container, cmd: Union[str, List[str]], ) -> Tuple[int, str]: retcode, output = ctr.exec_run(cmd) return retcode, output.decode("utf8").strip()
def test_pre_commit_presence( container: docker.models.containers.Container) -> None: """Is pre-commit installed?""" res: docker.ExecResult = container.exec_run("pre-commit --version") assert res.exit_code == 0, f"pre-commit not correctly or not installed in container"
def container_pid(c: docker.models.containers.Container) -> int: """ returns PID of `c` according in the host namespace """ return int(c.top()["Processes"][0][1])