def test_invalid_arguments_passed_to_container_to_handle(self, argv):
with patch('cdflow.docker') as docker, \
patch('cdflow.os') as os, \
patch('cdflow.open') as open_:
account_id = '1234567890'
config_file = MagicMock(spec=TextIOWrapper)
config_file.read.return_value = json.dumps(
{'platform_config': {
'account_id': account_id
}})
open_.return_value.__enter__.return_value = config_file
error = ContainerError(container=CDFLOW_IMAGE_ID,
exit_status=1,
command=argv,
image=CDFLOW_IMAGE_ID,
stderr='help text')
docker.from_env.return_value.containers.run.side_effect = error
os.path.abspath.return_value = '/'
exit_status = main(argv)
assert exit_status == 1
docker.from_env.return_value.containers.run.assert_called_once_with(
CDFLOW_IMAGE_ID,
command=argv,
environment=ANY,
detach=True,
volumes=ANY,
working_dir=ANY)
def read(cls,
container: Container,
command: str = None,
with_ext: bool = True) -> Iterable[str]:
"""Reads connector's logs per line"""
buffer = b""
exception = ""
line = ""
for chunk in container.logs(stdout=True,
stderr=True,
stream=True,
follow=True):
buffer += chunk
while True:
# every chunk can include several lines
found = buffer.find(b"\n")
if found <= -1:
break
line = buffer[:found + 1].decode("utf-8")
if len(exception) > 0 or line.startswith(
"Traceback (most recent call last)"):
exception += line
else:
yield line
buffer = buffer[found + 1:]
if buffer:
# send the latest chunk if exists
line = buffer.decode("utf-8")
if exception:
exception += line
else:
yield line
try:
exit_status = container.wait()
container.remove()
except NotFound as err:
logging.error(f"Waiting error: {err}, logs: {exception or line}")
raise
if exit_status["StatusCode"]:
error = exit_status["Error"] or exception or line
logging.error(f"Docker container was failed, "
f'code {exit_status["StatusCode"]}, error:\n{error}')
if with_ext:
raise ContainerError(
container=container,
exit_status=exit_status["StatusCode"],
command=command,
image=container.image,
stderr=error,
)
def test_container_without_stderr(self):
"""The massage does not contain stderr"""
client = make_fake_client()
container = client.containers.get(FAKE_CONTAINER_ID)
command = "echo Hello World"
exit_status = 42
image = FAKE_IMAGE_ID
stderr = None
err = ContainerError(container, exit_status, command, image, stderr)
msg = ("Command '{}' in image '{}' returned non-zero exit status {}"
).format(command, image, exit_status, stderr)
assert str(err) == msg
def read(cls,
container: Container,
command: str = None,
with_ext: bool = True) -> Iterable[str]:
"""Reads connector's logs per line"""
buffer = b""
has_exception = False
for chunk in container.logs(stdout=True,
stderr=True,
stream=True,
follow=True):
buffer += chunk
found = buffer.find(b"\n")
if found <= -1:
continue
line = buffer[:found].decode("utf-8")
if has_exception or "Traceback (most recent call last)" in line:
has_exception = True
else:
yield line
buffer = buffer[found + 1:]
if not has_exception and buffer:
yield buffer.decode("utf-8")
exit_status = container.wait()
if exit_status["StatusCode"]:
error = buffer.decode(
"utf-8") if has_exception else exit_status["Error"]
logging.error(f"Docker container was failed, "
f'code {exit_status["StatusCode"]}, error:\n{error}')
if with_ext:
raise ContainerError(
container=container,
exit_status=exit_status["StatusCode"],
command=command,
image=container.image,
stderr=error,
)
def run(self,
cmd,
config=None,
state=None,
catalog=None,
**kwargs) -> Iterable[AirbyteMessage]:
self._runs += 1
volumes = self._prepare_volumes(config, state, catalog)
logging.info("Docker run: \n%s\ninput: %s\noutput: %s", cmd,
self.input_folder, self.output_folder)
try:
logs = self._client.containers.run(image=self._image,
command=cmd,
working_dir="/data",
volumes=volumes,
network="host",
stdout=True,
stderr=True,
**kwargs)
except ContainerError as err:
# beautify error from container
patched_error = ContainerError(container=err.container,
exit_status=err.exit_status,
command=err.command,
image=err.image,
stderr=err.stderr.decode())
raise patched_error from None # get rid of any previous exception stack
with open(str(self.output_folder / "raw"), "wb+") as f:
f.write(logs)
for line in logs.decode("utf-8").splitlines():
try:
yield AirbyteMessage.parse_raw(line)
except ValidationError as exc:
logging.warning("Unable to parse connector's output %s", exc)
image=ANY,
command=ANY,
auto_remove=True,
volumes={str(tmp_path): {
"bind": "/project",
"mode": "rw"
}},
stream=True,
user=ANY,
)
@pytest.mark.parametrize(
"exception",
[
lambda: ContainerError("abcde", 255, "/bin/false", "image", ""),
ImageLoadError,
lambda: APIError("500"),
lambda: RequestsConnectionError(
"Connection aborted.",
ConnectionRefusedError(61, "Connection refused")),
],
)
def test__docker_build_bad_path(plugin, tmp_path, exception):
patch_sdist = patch.object(PythonLanguagePlugin,
"_check_for_support_lib_sdist")
patch_from_env = patch("rpdk.python.codegen.docker.from_env",
autospec=True)
with patch_sdist as mock_sdist, patch_from_env as mock_from_env:
mock_run = mock_from_env.return_value.containers.run
def run(self):
self.before_run()
# get image if missing
if self.force_pull or len(self._client.images(name=self._image)) == 0:
logger.info("Pulling docker image " + self._image)
try:
for logline in self._client.pull(self._image, stream=True):
logger.debug(logline.decode("utf-8"))
except APIError as e:
self.__logger.warning("Error in Docker API: " + e.explanation)
raise
# remove clashing container if a container with the same name exists
if self.auto_remove and self.name:
try:
self._client.remove_container(self.name, force=True)
except APIError as e:
self.__logger.warning("Ignored error in Docker API: " +
e.explanation)
# run the container
try:
logger.debug("Creating image: %s command: %s volumes: %s" %
(self._image, self.command, self._binds))
host_config = self._client.create_host_config(
binds=self._binds,
network_mode=self.network_mode,
auto_remove=self.auto_remove,
**self.host_config_options)
container = self._client.create_container(
self._image,
command=self.command,
name=self.name,
environment=self.environment,
volumes=self._volumes,
host_config=host_config,
**self.container_options)
self._client.start(container["Id"])
exit_status = self._client.wait(container["Id"])
# docker-py>=3.0.0 returns a dict instead of the status code directly
if type(exit_status) is dict:
exit_status = exit_status["StatusCode"]
if exit_status != 0:
stdout = False
stderr = True
error = self._client.logs(container["Id"],
stdout=stdout,
stderr=stderr)
if self.auto_remove:
try:
self._client.remove_container(container["Id"], force=True)
except docker.errors.APIError:
self.__logger.warning("Container " + container["Id"] +
" could not be removed")
if exit_status != 0:
raise ContainerError(container, exit_status, self.command,
self._image, error)
except ContainerError as e:
# catch non zero exti status and return it
container_name = ""
if self.name:
container_name = self.name
try:
message = e.message
except AttributeError:
message = str(e)
self.__logger.error("Container " + container_name +
" exited with non zero code: " + message)
raise
except ImageNotFound as e:
self.__logger.error("Image " + self._image + " not found")
raise
except APIError as e:
self.__logger.error("Error in Docker API: " + e.explanation)
raise
self.after_run()
def raise_containererror(*args, **kwargs):
raise ContainerError('container', 'exit_status', 'command',
'image', 'stderr')
def run(self):
# get image if missing
if self.force_pull or len(self._client.images(name=self._image)) == 0:
logger.info('Pulling docker image ' + self._image)
try:
for logline in self._client.pull(self._image, stream=True):
logger.debug(logline.decode('utf-8'))
except APIError as e:
self.__logger.warning("Error in Docker API: " + e.explanation)
raise
# remove clashing container if a container with the same name exists
if self.auto_remove and self.name:
try:
self._client.remove_container(self.name, force=True)
except APIError as e:
self.__logger.warning("Ignored error in Docker API: " +
e.explanation)
# run the container
try:
logger.debug('Creating image: %s command: %s volumes: %s' %
(self._image, self.command, self._binds))
host_config = self._client.create_host_config(
binds=self._binds, network_mode=self.network_mode)
container = self._client.create_container(
self._image,
command=self.command,
name=self.name,
environment=self.environment,
volumes=self._volumes,
host_config=host_config,
**self.container_options)
self._client.start(container['Id'])
exit_status = self._client.wait(container['Id'])
if exit_status != 0:
stdout = False
stderr = True
error = self._client.logs(container['Id'],
stdout=stdout,
stderr=stderr)
if self.auto_remove:
try:
self._client.remove_container(container['Id'])
except docker.errors.APIError:
self.__logger.warning("Container " + container['Id'] +
" could not be removed")
if exit_status != 0:
raise ContainerError(container, exit_status, self.command,
self._image, error)
except ContainerError as e:
# catch non zero exti status and return it
container_name = ''
if self.name:
container_name = self.name
try:
message = e.message
except:
message = str(e)
self.__logger.error("Container " + container_name +
" exited with non zero code: " + message)
raise
except ImageNotFound as e:
self.__logger.error("Image " + self._image + " not found")
raise
except APIError as e:
self.__logger.error("Error in Docker API: " + e.explanation)
raise
# delete temp dir
filesys = LocalFileSystem()
if self.mount_tmp and filesys.exists(self._host_tmp_dir):
filesys.remove(self._host_tmp_dir, recursive=True)
options["volumes"] = {}
volumes = options["volumes"]
volumes[tempdir] = {"bind": "/run", "mode": "rw"}
if "working_dir" not in options:
options["working_dir"] = "/run"
full_docker_command = "bash DOCKER-COMMAND"
from docker.types import LogConfig
container = docker_client.containers.create(docker_image,
full_docker_command, **options)
stdout0 = container.attach(stdout=True, stderr=False, stream=True)
container.start()
exit_status = container.wait()['StatusCode']
container.remove()
if exit_status != 0:
from docker.errors import ContainerError
stderr = container.logs(stdout=False, stderr=True)
raise ContainerError(container, exit_status, full_docker_command,
docker_image, stderr)
stdout = None if stdout0 is None else b''.join([line for line in stdout0])
os.chdir(cwd)
with open("test-docker-api-RESULT.npy", "wb") as f:
f.write(stdout)
import numpy as np
np.save("test-docker-api-ORIGINAL.npy", np.arange(12) * 3)
finally:
com = "rm -rf {}".format(os.path.abspath(tempdir))
os.system(com)