def execute(self, callFunc):
import paramiko
from docker import DockerClient
from sshtunnel import SSHTunnelForwarder
import time
for host in self.adress:
user = host.split('@', 1)[0]
addr = host.split('@', 1)[1]
port = self.localConnectionPort
forward = SSHTunnelForwarder(
(addr, 22),
ssh_username=user,
ssh_pkey="/var/ssh/rsa_key",
ssh_private_key_password="******",
remote_bind_address=(addr, 2375),
local_bind_address=('127.0.0.1', port))
forward.start()
time.sleep(3)
dockerConection = DockerClient(
'tcp://127.0.0.1:{port}'.format(port=port))
res = callFunc(dockerConection)
dockerConection.close()
del dockerConection
forward.stop()
return res
class TestSystem(unittest.TestCase):
podman = None # initialized podman configuration for tests
service = None # podman service instance
topContainerId = ""
def setUp(self):
super().setUp()
self.client = DockerClient(base_url="tcp://127.0.0.1:8080", timeout=15)
TestSystem.podman.restore_image_from_cache(self.client)
TestSystem.topContainerId = common.run_top_container(self.client)
def tearDown(self):
common.remove_all_containers(self.client)
common.remove_all_images(self.client)
self.client.close()
return super().tearDown()
@classmethod
def setUpClass(cls):
super().setUpClass()
TestSystem.podman = Podman()
TestSystem.service = TestSystem.podman.open("system", "service",
"tcp:127.0.0.1:8080",
"--time=0")
# give the service some time to be ready...
time.sleep(2)
returncode = TestSystem.service.poll()
if returncode is not None:
raise subprocess.CalledProcessError(returncode,
"podman system service")
@classmethod
def tearDownClass(cls):
TestSystem.service.terminate()
stdout, stderr = TestSystem.service.communicate(timeout=0.5)
if stdout:
sys.stdout.write("\nImages Service Stdout:\n" +
stdout.decode("utf-8"))
if stderr:
sys.stderr.write("\nImAges Service Stderr:\n" +
stderr.decode("utf-8"))
TestSystem.podman.tear_down()
return super().tearDownClass()
def test_Info(self):
self.assertIsNotNone(self.client.info())
def test_info_container_details(self):
info = self.client.info()
self.assertEqual(info["Containers"], 1)
self.client.containers.create(image=constant.ALPINE)
info = self.client.info()
self.assertEqual(info["Containers"], 2)
def test_version(self):
version = self.client.version()
self.assertIsNotNone(version["Platform"]["Name"])
def start():
sleep(config.container_start_period)
while True:
try:
dc = DockerClient(
base_url=config.docker_base_url,
timeout=config.docker_timeout
)
for container in dc.containers.list(
filters={"health": "unhealthy", "label": [f"{config.container_label}=true"]}):
process_container(container)
sleep(config.container_interval)
except ReadTimeout as e:
log(f"<7ba9d80d> Connection timed out. Restart containers processing", LogLevel.ERROR)
except Exception as e:
log(f"<4be316f> Error: {e}", LogLevel.ERROR)
finally:
dc.close()
class TestContainers(unittest.TestCase):
podman = None # initialized podman configuration for tests
service = None # podman service instance
topContainerId = ""
def setUp(self):
super().setUp()
self.client = DockerClient(base_url="tcp://127.0.0.1:8080", timeout=15)
TestContainers.podman.restore_image_from_cache(self.client)
TestContainers.topContainerId = common.run_top_container(self.client)
self.assertIsNotNone(TestContainers.topContainerId)
def tearDown(self):
common.remove_all_containers(self.client)
common.remove_all_images(self.client)
self.client.close()
return super().tearDown()
@classmethod
def setUpClass(cls):
super().setUpClass()
TestContainers.podman = Podman()
TestContainers.service = TestContainers.podman.open(
"system", "service", "tcp:127.0.0.1:8080", "--time=0")
# give the service some time to be ready...
time.sleep(2)
rc = TestContainers.service.poll()
if rc is not None:
raise subprocess.CalledProcessError(rc, "podman system service")
@classmethod
def tearDownClass(cls):
TestContainers.service.terminate()
stdout, stderr = TestContainers.service.communicate(timeout=0.5)
if stdout:
sys.stdout.write("\nContainers Service Stdout:\n" +
stdout.decode("utf-8"))
if stderr:
sys.stderr.write("\nContainers Service Stderr:\n" +
stderr.decode("utf-8"))
TestContainers.podman.tear_down()
return super().tearDownClass()
def test_create_container(self):
# Run a container with detach mode
self.client.containers.create(image="alpine", detach=True)
self.assertEqual(len(self.client.containers.list(all=True)), 2)
def test_create_network(self):
net = self.client.networks.create("testNetwork", driver="bridge")
ctnr = self.client.containers.create(image="alpine", detach=True)
# TODO fix when ready
# This test will not work until all connect|disconnect
# code is fixed.
# net.connect(ctnr)
# nets = self.client.networks.list(greedy=True)
# self.assertGreaterEqual(len(nets), 1)
# TODO fix endpoint to include containers
# for n in nets:
# if n.id == "testNetwork":
# self.assertEqual(ctnr.id, n.containers)
# self.assertTrue(False, "testNetwork not found")
def test_start_container(self):
# Podman docs says it should give a 304 but returns with no response
# # Start a already started container should return 304
# response = self.client.api.start(container=TestContainers.topContainerId)
# self.assertEqual(error.exception.response.status_code, 304)
# Create a new container and validate the count
self.client.containers.create(image=constant.ALPINE, name="container2")
containers = self.client.containers.list(all=True)
self.assertEqual(len(containers), 2)
def test_start_container_with_random_port_bind(self):
container = self.client.containers.create(
image=constant.ALPINE,
name="containerWithRandomBind",
ports={"1234/tcp": None},
)
containers = self.client.containers.list(all=True)
self.assertTrue(container in containers)
def test_stop_container(self):
top = self.client.containers.get(TestContainers.topContainerId)
self.assertEqual(top.status, "running")
# Stop a running container and validate the state
top.stop()
top.reload()
self.assertIn(top.status, ("stopped", "exited"))
def test_kill_container(self):
top = self.client.containers.get(TestContainers.topContainerId)
self.assertEqual(top.status, "running")
# Kill a running container and validate the state
top.kill()
top.reload()
self.assertIn(top.status, ("stopped", "exited"))
def test_restart_container(self):
# Validate the container state
top = self.client.containers.get(TestContainers.topContainerId)
top.stop()
top.reload()
self.assertIn(top.status, ("stopped", "exited"))
# restart a running container and validate the state
top.restart()
top.reload()
self.assertEqual(top.status, "running")
def test_remove_container(self):
# Remove container by ID with force
top = self.client.containers.get(TestContainers.topContainerId)
top.remove(force=True)
self.assertEqual(len(self.client.containers.list()), 0)
def test_remove_container_without_force(self):
# Validate current container count
self.assertEqual(len(self.client.containers.list()), 1)
# Remove running container should throw error
top = self.client.containers.get(TestContainers.topContainerId)
with self.assertRaises(errors.APIError) as error:
top.remove()
self.assertEqual(error.exception.response.status_code, 500)
# Remove container by ID without force
top.stop()
top.remove()
self.assertEqual(len(self.client.containers.list()), 0)
def test_pause_container(self):
# Validate the container state
top = self.client.containers.get(TestContainers.topContainerId)
self.assertEqual(top.status, "running")
# Pause a running container and validate the state
top.pause()
top.reload()
self.assertEqual(top.status, "paused")
def test_pause_stopped_container(self):
# Stop the container
top = self.client.containers.get(TestContainers.topContainerId)
top.stop()
# Pause exited container should throw error
with self.assertRaises(errors.APIError) as error:
top.pause()
self.assertEqual(error.exception.response.status_code, 500)
def test_unpause_container(self):
top = self.client.containers.get(TestContainers.topContainerId)
# Validate the container state
top.pause()
top.reload()
self.assertEqual(top.status, "paused")
# Pause a running container and validate the state
top.unpause()
top.reload()
self.assertEqual(top.status, "running")
def test_list_container(self):
# Add container and validate the count
self.client.containers.create(image="alpine", detach=True)
containers = self.client.containers.list(all=True)
self.assertEqual(len(containers), 2)
def test_filters(self):
self.skipTest("TODO Endpoint does not yet support filters")
# List container with filter by id
filters = {"id": TestContainers.topContainerId}
ctnrs = self.client.containers.list(all=True, filters=filters)
self.assertEqual(len(ctnrs), 1)
# List container with filter by name
filters = {"name": "top"}
ctnrs = self.client.containers.list(all=True, filters=filters)
self.assertEqual(len(ctnrs), 1)
def test_copy_to_container(self):
ctr: Optional[Container] = None
try:
test_file_content = b"Hello World!"
ctr = self.client.containers.create(image="alpine",
detach=True,
command="top")
ctr.start()
buff: IO[bytes] = io.BytesIO()
with tarfile.open(fileobj=buff, mode="w:") as tf:
ti: tarfile.TarInfo = tarfile.TarInfo()
ti.uid = 1042
ti.gid = 1043
ti.name = "a.txt"
ti.path = "a.txt"
ti.mode = 0o644
ti.type = tarfile.REGTYPE
ti.size = len(test_file_content)
tf.addfile(ti, fileobj=io.BytesIO(test_file_content))
buff.seek(0)
ctr.put_archive("/tmp/", buff)
ret, out = ctr.exec_run(["stat", "-c", "%u:%g", "/tmp/a.txt"])
self.assertEqual(ret, 0)
self.assertEqual(out.rstrip(), b'1042:1043',
"UID/GID of copied file")
ret, out = ctr.exec_run(["cat", "/tmp/a.txt"])
self.assertEqual(ret, 0)
self.assertEqual(out.rstrip(), test_file_content,
"Content of copied file")
finally:
if ctr is not None:
ctr.stop()
ctr.remove()
def test_mount_preexisting_dir(self):
dockerfile = (B'FROM quay.io/libpod/alpine:latest\n'
B'USER root\n'
B'RUN mkdir -p /workspace\n'
B'RUN chown 1042:1043 /workspace')
img: Image
img, out = self.client.images.build(fileobj=io.BytesIO(dockerfile))
ctr: Container = self.client.containers.create(
image=img.id,
detach=True,
command="top",
volumes=["test_mount_preexisting_dir_vol:/workspace"])
ctr.start()
ret, out = ctr.exec_run(["stat", "-c", "%u:%g", "/workspace"])
self.assertEqual(out.rstrip(), b'1042:1043',
"UID/GID set in dockerfile")
class TestContainers(unittest.TestCase):
podman = None # initialized podman configuration for tests
service = None # podman service instance
topContainerId = ""
def setUp(self):
super().setUp()
self.client = DockerClient(base_url="tcp://127.0.0.1:8080", timeout=15)
TestContainers.podman.restore_image_from_cache(self.client)
TestContainers.topContainerId = common.run_top_container(self.client)
self.assertIsNotNone(TestContainers.topContainerId)
def tearDown(self):
common.remove_all_containers(self.client)
common.remove_all_images(self.client)
self.client.close()
return super().tearDown()
@classmethod
def setUpClass(cls):
super().setUpClass()
TestContainers.podman = Podman()
TestContainers.service = TestContainers.podman.open(
"system", "service", "tcp:127.0.0.1:8080", "--time=0"
)
# give the service some time to be ready...
time.sleep(2)
rc = TestContainers.service.poll()
if rc is not None:
raise subprocess.CalledProcessError(rc, "podman system service")
@classmethod
def tearDownClass(cls):
TestContainers.service.terminate()
stdout, stderr = TestContainers.service.communicate(timeout=0.5)
if stdout:
sys.stdout.write("\nContainers Service Stdout:\n" + stdout.decode("utf-8"))
if stderr:
sys.stderr.write("\nContainers Service Stderr:\n" + stderr.decode("utf-8"))
TestContainers.podman.tear_down()
return super().tearDownClass()
def test_create_container(self):
# Run a container with detach mode
self.client.containers.create(image="alpine", detach=True)
self.assertEqual(len(self.client.containers.list(all=True)), 2)
def test_create_network(self):
net = self.client.networks.create("testNetwork", driver="bridge")
ctnr = self.client.containers.create(image="alpine", detach=True)
# TODO fix when ready
# This test will not work until all connect|disconnect
# code is fixed.
# net.connect(ctnr)
# nets = self.client.networks.list(greedy=True)
# self.assertGreaterEqual(len(nets), 1)
# TODO fix endpoint to include containers
# for n in nets:
# if n.id == "testNetwork":
# self.assertEqual(ctnr.id, n.containers)
# self.assertTrue(False, "testNetwork not found")
def test_start_container(self):
# Podman docs says it should give a 304 but returns with no response
# # Start a already started container should return 304
# response = self.client.api.start(container=TestContainers.topContainerId)
# self.assertEqual(error.exception.response.status_code, 304)
# Create a new container and validate the count
self.client.containers.create(image=constant.ALPINE, name="container2")
containers = self.client.containers.list(all=True)
self.assertEqual(len(containers), 2)
def test_stop_container(self):
top = self.client.containers.get("top")
self.assertEqual(top.status, "running")
# Stop a running container and validate the state
top.stop()
top.reload()
self.assertIn(top.status, ("stopped", "exited"))
def test_restart_container(self):
# Validate the container state
top = self.client.containers.get(TestContainers.topContainerId)
top.stop()
top.reload()
self.assertIn(top.status, ("stopped", "exited"))
# restart a running container and validate the state
top.restart()
top.reload()
self.assertEqual(top.status, "running")
def test_remove_container(self):
# Remove container by ID with force
top = self.client.containers.get(TestContainers.topContainerId)
top.remove(force=True)
self.assertEqual(len(self.client.containers.list()), 0)
def test_remove_container_without_force(self):
# Validate current container count
self.assertTrue(len(self.client.containers.list()), 1)
# Remove running container should throw error
top = self.client.containers.get(TestContainers.topContainerId)
with self.assertRaises(errors.APIError) as error:
top.remove()
self.assertEqual(error.exception.response.status_code, 500)
# Remove container by ID without force
top.stop()
top.remove()
self.assertEqual(len(self.client.containers.list()), 0)
def test_pause_container(self):
# Validate the container state
top = self.client.containers.get(TestContainers.topContainerId)
self.assertEqual(top.status, "running")
# Pause a running container and validate the state
top.pause()
top.reload()
self.assertEqual(top.status, "paused")
def test_pause_stopped_container(self):
# Stop the container
top = self.client.containers.get(TestContainers.topContainerId)
top.stop()
# Pause exited container should trow error
with self.assertRaises(errors.APIError) as error:
top.pause()
self.assertEqual(error.exception.response.status_code, 500)
def test_unpause_container(self):
top = self.client.containers.get(TestContainers.topContainerId)
# Validate the container state
top.pause()
top.reload()
self.assertEqual(top.status, "paused")
# Pause a running container and validate the state
top.unpause()
top.reload()
self.assertEqual(top.status, "running")
def test_list_container(self):
# Add container and validate the count
self.client.containers.create(image="alpine", detach=True)
containers = self.client.containers.list(all=True)
self.assertEqual(len(containers), 2)
def test_filters(self):
self.skipTest("TODO Endpoint does not yet support filters")
# List container with filter by id
filters = {"id": TestContainers.topContainerId}
ctnrs = self.client.containers.list(all=True, filters=filters)
self.assertEqual(len(ctnrs), 1)
# List container with filter by name
filters = {"name": "top"}
ctnrs = self.client.containers.list(all=True, filters=filters)
self.assertEqual(len(ctnrs), 1)
def test_rename_container(self):
top = self.client.containers.get(TestContainers.topContainerId)
# rename bogus container
with self.assertRaises(errors.APIError) as error:
top.rename(name="newname")
self.assertEqual(error.exception.response.status_code, 404)
class TestImages(unittest.TestCase):
podman = None # initialized podman configuration for tests
service = None # podman service instance
def setUp(self):
super().setUp()
self.client = DockerClient(base_url="tcp://127.0.0.1:8080", timeout=15)
TestImages.podman.restore_image_from_cache(self.client)
def tearDown(self):
common.remove_all_images(self.client)
self.client.close()
return super().tearDown()
@classmethod
def setUpClass(cls):
super().setUpClass()
TestImages.podman = Podman()
TestImages.service = TestImages.podman.open(
"system", "service", "tcp:127.0.0.1:8080", "--time=0"
)
# give the service some time to be ready...
time.sleep(2)
returncode = TestImages.service.poll()
if returncode is not None:
raise subprocess.CalledProcessError(returncode, "podman system service")
@classmethod
def tearDownClass(cls):
TestImages.service.terminate()
stdout, stderr = TestImages.service.communicate(timeout=0.5)
if stdout:
sys.stdout.write("\nImages Service Stdout:\n" + stdout.decode("utf-8"))
if stderr:
sys.stderr.write("\nImAges Service Stderr:\n" + stderr.decode("utf-8"))
TestImages.podman.tear_down()
return super().tearDownClass()
def test_tag_valid_image(self):
"""Validates if the image is tagged successfully"""
alpine = self.client.images.get(constant.ALPINE)
self.assertTrue(alpine.tag("demo", constant.ALPINE_SHORTNAME))
alpine = self.client.images.get(constant.ALPINE)
for t in alpine.tags:
self.assertIn("alpine", t)
# @unittest.skip("doesn't work now")
def test_retag_valid_image(self):
"""Validates if name updates when the image is retagged"""
alpine = self.client.images.get(constant.ALPINE)
self.assertTrue(alpine.tag("demo", "rename"))
alpine = self.client.images.get(constant.ALPINE)
self.assertNotIn("demo:test", alpine.tags)
def test_list_images(self):
"""List images"""
self.assertEqual(len(self.client.images.list()), 1)
# Add more images
self.client.images.pull(constant.BB)
self.assertEqual(len(self.client.images.list()), 2)
self.assertEqual(len(self.client.images.list(all=True)), 2)
# List images with filter
self.assertEqual(len(self.client.images.list(filters={"reference": "alpine"})), 1)
def test_search_image(self):
"""Search for image"""
for r in self.client.images.search("alpine"):
# registry matches if string is in either one
self.assertIn("alpine", r["Name"]+" "+r["Description"].lower())
def test_search_bogus_image(self):
"""Search for bogus image should throw exception"""
try:
r = self.client.images.search("bogus/bogus")
except:
return
self.assertTrue(len(r) == 0)
def test_remove_image(self):
"""Remove image"""
# Check for error with wrong image name
with self.assertRaises(errors.NotFound):
self.client.images.remove("dummy")
self.assertEqual(len(self.client.images.list()), 1)
self.client.images.remove(constant.ALPINE)
self.assertEqual(len(self.client.images.list()), 0)
def test_image_history(self):
"""Image history"""
img = self.client.images.get(constant.ALPINE)
history = img.history()
image_id = img.id[7:] if img.id.startswith("sha256:") else img.id
found = False
for change in history:
found |= image_id in change.values()
self.assertTrue(found, f"image id {image_id} not found in history")
def test_get_image_exists_not(self):
"""Negative test for get image"""
with self.assertRaises(errors.NotFound):
response = self.client.images.get("image_does_not_exists")
collections.deque(response)
def test_save_image(self):
"""Export Image"""
image = self.client.images.pull(constant.BB)
file = os.path.join(TestImages.podman.image_cache, "busybox.tar")
with open(file, mode="wb") as tarball:
for frame in image.save(named=True):
tarball.write(frame)
sz = os.path.getsize(file)
self.assertGreater(sz, 0)
def test_load_image(self):
"""Import|Load Image"""
self.assertEqual(len(self.client.images.list()), 1)
image = self.client.images.pull(constant.BB)
file = os.path.join(TestImages.podman.image_cache, "busybox.tar")
with open(file, mode="wb") as tarball:
for frame in image.save():
tarball.write(frame)
with open(file, mode="rb") as saved:
_ = self.client.images.load(saved)
self.assertEqual(len(self.client.images.list()), 2)
def test_load_corrupt_image(self):
"""Import|Load Image failure"""
tarball = io.BytesIO("This is a corrupt tarball".encode("utf-8"))
with self.assertRaises(APIError):
self.client.images.load(tarball)
def test_build_image(self):
labels = {"apple": "red", "grape": "green"}
_ = self.client.images.build(
path="test/python/docker/build_labels", labels=labels, tag="labels", isolation="default"
)
image = self.client.images.get("labels")
self.assertEqual(image.labels["apple"], labels["apple"])
self.assertEqual(image.labels["grape"], labels["grape"])
#!/usr/bin/env python3 from docker import DockerClient import config docker_client = DockerClient(base_url=config.docker_base_url) docker_client.ping() docker_client.close() exit(0)
class DockerManager():
def __init__(self, docker_url=None, verbose=False):
self._client = DockerClient(base_url=docker_url)
self._docker_url = docker_url
self._verbose = verbose
try:
self._client.ping()
except APIError as err:
logger.exception(err)
raise DockerOperationError(
'Failed to connect to the Docker Engine.')
def get_docker_info(self):
""" Returns info about the local docker engine """
try:
return {
'info': self._client.info(),
'version': self._client.version(),
}
except APIError as err:
logger.exception(err)
raise DockerOperationError(
'Failed to retrieve the Docker Engine info')
def search_runnable_commands(self, image, tag, pull=True):
""" Searching for ENTRYPOINT or CMD in a given image """
def _bind_shell(cmd_list):
""" e.g. ["/bin/sh", "-c", "echo", "hello!"] """
if cmd_list[0] in SUPPORTED_SHELLS:
return cmd_list
else:
# add a default shell on head
return (DEFAULT_SHELL + cmd_list)
full_name = '{0}:{1}'.format(image, tag)
logger.info('Analyzing the [{}] image'.format(full_name))
image = self._pull_image_with_auth(image, tag)
if image is not None:
output = self._client.api.inspect_image(full_name)
if not output or 'Config' not in output:
raise FatalError(
'Failed to inspect {} image'.format(full_name))
else:
entrypoint = output['Config']['Entrypoint']
cmd = output['Config']['Cmd']
commands = list()
if entrypoint and not cmd:
logger.debug(
'Detected only an ENTRYPOINT: {}'.format(entrypoint))
commands = _bind_shell(entrypoint)
elif not entrypoint and cmd:
logger.debug('Detected only a CMD: {}'.format(cmd))
commands = _bind_shell(cmd)
elif entrypoint and cmd:
logger.debug('Detected both ENTRYPOINT: \
{0} and CMD: {1}'.format(entrypoint, cmd))
# combining entrypoint (first) with cmd
commands = _bind_shell(entrypoint)
commands += cmd
else:
logger.info('the {} image is not runnable. \
Adding an infinite sleep'.format(full_name))
commands = DEFAULT_SHELL + ['sleep', 'infinity']
# add quotes
# [0]: /bin/bash [1]: -c
commands[2] = "\'" + commands[2]
commands[-1] = commands[-1] + "\'"
return ' '.join(commands)
def _pull_image_with_auth(self, image, tag):
try:
return self._client.images.pull(image, tag)
except ImageNotFound:
# it should be an authentication error
return self._image_authentication(image, tag)
def _image_authentication(self, src_image, src_tag=None, auth=None):
""" Handling Docker authentication if the image
is hosted on a private repository
Args:
src_image (str): The original image that needs
authentication for being fetched.
src_tag (str): The tag of the image above.
"""
if src_tag is None:
src_tag = 'latest'
logger.warning('[{0}:{1}] image may not exist or authentication \
is required'.format(src_image, src_tag))
res = ''
while res != 'YES':
res = (input('[{0}:{1}] is correct? [Yes] Continue \
[No] Abort\n'.format(src_image, src_tag))).upper()
if res == 'NO':
logger.error(
'Docker image [{0}:{1}] cannot be found, the operation \
is aborted by the user.\n(Hint: Check the TOSCA manifest.)'
.format(src_image, src_tag))
raise OperationAbortedByUser(
CommonErrorMessages._DEFAULT_OPERATION_ABORTING_ERROR_MSG)
attempts = 3
while attempts > 0:
logger.info('Authenticate with the Docker Repository..')
try:
if auth is None:
auth = create_auth_interactive(
user_text='Enter the username: '******'Enter the password: '******'UNAUTHORIZED' or 'NOT FOUND' in msg:
logger.info('Invalid username/password.')
else:
logger.exception(err)
raise FatalError(
CommonErrorMessages._DEFAULT_FATAL_ERROR_MSG)
attempts -= 1
logger.info('Authentication failed. \
You have [{}] more attempts.'.format(attempts))
# Check authentication failure
if attempts == 0:
logger.error(
'You have used all the authentication attempts. Abort.')
raise DockerAuthenticationFailedError(
'Authentication failed. Abort.')
def _push_image(self, image, tag=None, auth=None, attempts=0):
""" Push an image to a remote Docker Registry.
Args:
image (str): The name of the Docker Image to be pushed.
tag (str): An optional tag for the Docker Image (default: 'latest')
auth: An optional Dict for the authentication.
attempts (int): The number of unsuccessful authentication attempts.
"""
if tag is None:
tag = 'latest'
if attempts == MAX_PUSH_ATTEMPTS:
err = 'Reached max attempts for pushing \
[{0}] with tag [{1}]'.format(image, tag)
logger.error(err)
raise FatalError(err)
logger.info('Pushing [{0}] with tag [{1}]'.format(image, tag))
result = self._client.images.push(image,
tag=tag,
auth_config=auth,
stream=True,
decode=True)
# Access Denied detection
for line in result:
# an error is occurred
if 'error' in line:
# access denied
if 'denied' or 'unauthorized' in line['error']:
logger.info('Access to the repository denied. \
Authentication failed.')
auth = create_auth_interactive()
self._push_image(image,
tag=tag,
auth=auth,
attempts=(attempts + 1))
else:
logger.error(
'Unknown error during push of [{0}]: {1}'.format(
image, line['error']))
raise FatalError(
CommonErrorMessages._DEFAULT_FATAL_ERROR_MSG)
logger.info('Image [{0}] with tag [{1}] successfully pushed.'.format(
image, tag))
def _validate_build(self, build_logs):
""" Validate the output of the building process.
Args:
build_logs (str): The output logs of a docker
image building process.
"""
for log in build_logs:
if 'stream' in log and self._verbose:
print(log['stream'])
if 'error' in log:
logger.error('Error building the Docker Image:\
\n{}'.format(log['error']))
raise DockerOperationError('Failed to "toskosing" \
the Docker Image. Abort.')
def _toskose_image_availability(self, toskose_image, toskose_tag='latest'):
""" Check the availability of the official Docker Toskose image used
during the "toskosing" process.
Args:
toskose_image (str): The official Toskose Docker image.
toskose_tag (str): The tag of the image (default: 'latest').
"""
try:
_ = self._client.images.pull(toskose_image, tag=toskose_tag)
except ImageNotFound:
logger.error(
'Failed to retrieve the official Toskose image [{0}:{1}]. \
Abort'.format(toskose_image, toskose_tag))
raise DockerOperationError(
'The official Toskose image {0} not found. Try later.\
\nIf the problem persist, open an issue at {1}'.format(
toskose_image, toskose_tag))
def _remove_previous_toskosed(self, image, tag=None):
""" Remove previous toskosed images.
Note: docker rmi doesn't remove an image if there are multiple tags
referencing it.
Args:
image (str): The name of the Docker image.
tag (str): The tag of the Docker Image (default: 'latest').
"""
if tag is None:
tag = 'latest'
def print_well(tags):
out = ''
for tag in tags:
out += '- {}\n'.format(tag)
return out
try:
logger.info(
'Searching for previous toskosed images [{0}:{1}]'.format(
image, tag))
image_found = self._client.images.get(image)
logger.debug(
'Image [{0}] found. It\'s referenced by the following \
tags:\n\n{1}'.format(image, print_well(image_found.tags)))
full_name = '{0}:{1}'.format(image, tag)
if full_name in image_found.tags:
self._client.images.remove(image=full_name, force=True)
logger.info('Removed [{0}] reference from [{1}] image'.format(
full_name, image))
try:
image_found = self._client.images.get(image)
logger.debug('Image [{0}][ID: {1}] still exist'.format(
image, image_found.id))
except ImageNotFound:
logger.info('Image [{0}][ID: {1}] doesn\'t have any \
references yet. Removed.'.format(
image, image_found.id))
except ImageNotFound:
logger.info('No previous image found.')
def toskose_image(self,
src_image,
src_tag,
dst_image,
dst_tag,
context,
process_type,
app_name,
toskose_dockerfile=None,
toskose_image=None,
toskose_tag=None,
enable_push=True):
""" The process of "toskosing" the component(s) of a multi-component
TOSCA-defined application.
The "toskosing" process consists in merging contexts
(e.g. artifacts, lifecycle scripts) of TOSCA software component(s)
and a TOSCA container node in which they are hosted on in a new
docker image.
The TOSCA container node comes with a suggested docker image,
which is "enriched" by the Toskose logic fetched remotely from
the official Toskose Docker image, using a template Dockerfile
by means of docker multi-stage features.
This logic permits to handle the lifecycle of multiple components
within the same container. (More details on how this process works
can be found in the template Dockerfile in dockerfiles/ directory or
in the official Toskose GitHub repository)
In other words, this function consists in receiving a name of an
existing docker image and a path to a build context, then by means
of the docker client, the toskose template dockerfile is built and
a new fresh docker image is generated. The new docker image includes
the content of the original image, the contexts of the TOSCA software
components and the logic of toskose for managing them.
Args:
src_image (str): The name of the image to be "toskosed".
src_tag (str): The tag of the image to be "toskosed".
dst_image (str): The name of the "toskosed" image.
dst_tag (str): The tag of the "toskosed" image.
context (str): The path of the application context.
process_type (enum): The type of "toskosing" process.
[unit/manager/free]
app_name (str): The name of the TOSCA application.
toskose_dockerfile (str): The path of the template dockerfile
used in the process.
toskose_image (str): The Docker Toskose base-image used in the
"toskosing" process.
toskose_tag (str): The tag of the Docker Toskose base-image.
enable_push (bool): enable/disable pushing of the "toskosed" image.
(default: True)
"""
if toskose_dockerfile is not None:
if not os.path.exists(toskose_dockerfile):
raise ValueError('The given toskose template dockerfile \
{} doesn\'t exist'.format(toskose_dockerfile))
if not app_name:
raise ValueError('A name associated to the TOSCA application \
must be given')
template_dir = os.path.join(os.path.dirname(__file__),
DOCKERFILE_TEMPLATES_PATH)
# TODO can be enanched
if process_type == ToskosingProcessType.TOSKOSE_UNIT:
if toskose_image is None:
toskose_image = constants.DEFAULT_TOSKOSE_UNIT_BASE_IMAGE
if toskose_tag is None:
toskose_tag = constants.DEFAULT_TOSKOSE_UNIT_BASE_TAG
if toskose_dockerfile is None:
toskose_dockerfile = os.path.join(
template_dir, DOCKERFILE_TOSKOSE_UNIT_TEMPLATE)
elif process_type == ToskosingProcessType.TOSKOSE_MANAGER:
if toskose_image is None:
toskose_image = constants.DEFAULT_MANAGER_BASE_IMAGE
if toskose_tag is None:
toskose_tag = constants.DEFAULT_MANAGER_BASE_TAG
if toskose_dockerfile is None:
toskose_dockerfile = os.path.join(
template_dir, DOCKERFILE_TOSKOSE_MANAGER_TEMPLATE)
src_image = toskose_image
src_tag = toskose_tag
elif process_type == ToskosingProcessType.TOSKOSE_FREE:
pass
else:
raise ValueError('Cannot recognize the "toskosing" process \
{}'.format(process_type))
if toskose_image is not None:
self._toskose_image_availability(toskose_image, toskose_tag)
# Check if the original image exists and needs authentication
# to be fetched.
# note: docker client does not distinguish between authentication
# error or invalid image name (?)
logger.info('Pulling [{0}:{1}]'.format(src_image, src_tag))
self._pull_image_with_auth(src_image, src_tag)
# TODO can be removed? is it really necessary?
self._remove_previous_toskosed(dst_image, dst_tag)
logger.info('Toskosing [{0}:{1}] image'.format(src_image, src_tag))
try:
build_args = {
'TOSCA_SRC_IMAGE': '{0}:{1}'.format(src_image, src_tag),
'TOSKOSE_BASE_IMG': '{0}:{1}'.format(toskose_image,
toskose_tag)
}
image, build_logs = self._client.images.build(
path=context,
tag='{0}:{1}'.format(dst_image, dst_tag),
buildargs=build_args,
dockerfile=toskose_dockerfile,
rm=True, # remove intermediate containers
)
self._validate_build(build_logs)
# push the "toskosed" image
if enable_push:
self._push_image(dst_image, tag=dst_tag)
logger.info(
'[{0}:{1}] image successfully toskosed in [{2}:{3}].'.format(
src_image, src_tag, dst_image, dst_tag))
except (BuildError, APIError) as err:
logger.exception(err)
raise DockerOperationError('Failed to build image')
def close(self):
self._client.close()
from datetime import datetime, timezone
from docker import DockerClient
import dateutil.parser as parser
from docker.models.containers import Container
from requests.adapters import ReadTimeout
from execute_cmd import execute_cmd
from send_mail import notify_failure
from utils import log, LogLevel
import config
# Test docker connection
test_dc = DockerClient(base_url=config.docker_base_url)
test_dc.ping()
test_dc.close()
def restart_container(container: Container):
log(f"<27db178e> ({container.name}) Restaring container with timeout {config.container_stop_timeout}")
container.restart(timeout=config.container_stop_timeout)
def process_container(container: Container):
log(f"<193643d5> ({container.name}) Container seems to be unhealthy")
now = datetime.now(timezone.utc)
failure_time = container.attrs["State"]["Health"]["Log"][-1]["End"]
try:
parsed_time = parser.isoparse(failure_time)
if (now - parsed_time).seconds > config.container_debounce_time:
execute_cmd(container)
