def execute(self, callFunc): import paramiko from docker import DockerClient from sshtunnel import SSHTunnelForwarder import time for host in self.adress: user = host.split('@', 1)[0] addr = host.split('@', 1)[1] port = self.localConnectionPort forward = SSHTunnelForwarder( (addr, 22), ssh_username=user, ssh_pkey="/var/ssh/rsa_key", ssh_private_key_password="******", remote_bind_address=(addr, 2375), local_bind_address=('127.0.0.1', port)) forward.start() time.sleep(3) dockerConection = DockerClient( 'tcp://127.0.0.1:{port}'.format(port=port)) res = callFunc(dockerConection) dockerConection.close() del dockerConection forward.stop() return res
class TestSystem(unittest.TestCase): podman = None # initialized podman configuration for tests service = None # podman service instance topContainerId = "" def setUp(self): super().setUp() self.client = DockerClient(base_url="tcp://127.0.0.1:8080", timeout=15) TestSystem.podman.restore_image_from_cache(self.client) TestSystem.topContainerId = common.run_top_container(self.client) def tearDown(self): common.remove_all_containers(self.client) common.remove_all_images(self.client) self.client.close() return super().tearDown() @classmethod def setUpClass(cls): super().setUpClass() TestSystem.podman = Podman() TestSystem.service = TestSystem.podman.open("system", "service", "tcp:127.0.0.1:8080", "--time=0") # give the service some time to be ready... time.sleep(2) returncode = TestSystem.service.poll() if returncode is not None: raise subprocess.CalledProcessError(returncode, "podman system service") @classmethod def tearDownClass(cls): TestSystem.service.terminate() stdout, stderr = TestSystem.service.communicate(timeout=0.5) if stdout: sys.stdout.write("\nImages Service Stdout:\n" + stdout.decode("utf-8")) if stderr: sys.stderr.write("\nImAges Service Stderr:\n" + stderr.decode("utf-8")) TestSystem.podman.tear_down() return super().tearDownClass() def test_Info(self): self.assertIsNotNone(self.client.info()) def test_info_container_details(self): info = self.client.info() self.assertEqual(info["Containers"], 1) self.client.containers.create(image=constant.ALPINE) info = self.client.info() self.assertEqual(info["Containers"], 2) def test_version(self): version = self.client.version() self.assertIsNotNone(version["Platform"]["Name"])
def start(): sleep(config.container_start_period) while True: try: dc = DockerClient( base_url=config.docker_base_url, timeout=config.docker_timeout ) for container in dc.containers.list( filters={"health": "unhealthy", "label": [f"{config.container_label}=true"]}): process_container(container) sleep(config.container_interval) except ReadTimeout as e: log(f"<7ba9d80d> Connection timed out. Restart containers processing", LogLevel.ERROR) except Exception as e: log(f"<4be316f> Error: {e}", LogLevel.ERROR) finally: dc.close()
class TestContainers(unittest.TestCase): podman = None # initialized podman configuration for tests service = None # podman service instance topContainerId = "" def setUp(self): super().setUp() self.client = DockerClient(base_url="tcp://127.0.0.1:8080", timeout=15) TestContainers.podman.restore_image_from_cache(self.client) TestContainers.topContainerId = common.run_top_container(self.client) self.assertIsNotNone(TestContainers.topContainerId) def tearDown(self): common.remove_all_containers(self.client) common.remove_all_images(self.client) self.client.close() return super().tearDown() @classmethod def setUpClass(cls): super().setUpClass() TestContainers.podman = Podman() TestContainers.service = TestContainers.podman.open( "system", "service", "tcp:127.0.0.1:8080", "--time=0") # give the service some time to be ready... time.sleep(2) rc = TestContainers.service.poll() if rc is not None: raise subprocess.CalledProcessError(rc, "podman system service") @classmethod def tearDownClass(cls): TestContainers.service.terminate() stdout, stderr = TestContainers.service.communicate(timeout=0.5) if stdout: sys.stdout.write("\nContainers Service Stdout:\n" + stdout.decode("utf-8")) if stderr: sys.stderr.write("\nContainers Service Stderr:\n" + stderr.decode("utf-8")) TestContainers.podman.tear_down() return super().tearDownClass() def test_create_container(self): # Run a container with detach mode self.client.containers.create(image="alpine", detach=True) self.assertEqual(len(self.client.containers.list(all=True)), 2) def test_create_network(self): net = self.client.networks.create("testNetwork", driver="bridge") ctnr = self.client.containers.create(image="alpine", detach=True) # TODO fix when ready # This test will not work until all connect|disconnect # code is fixed. # net.connect(ctnr) # nets = self.client.networks.list(greedy=True) # self.assertGreaterEqual(len(nets), 1) # TODO fix endpoint to include containers # for n in nets: # if n.id == "testNetwork": # self.assertEqual(ctnr.id, n.containers) # self.assertTrue(False, "testNetwork not found") def test_start_container(self): # Podman docs says it should give a 304 but returns with no response # # Start a already started container should return 304 # response = self.client.api.start(container=TestContainers.topContainerId) # self.assertEqual(error.exception.response.status_code, 304) # Create a new container and validate the count self.client.containers.create(image=constant.ALPINE, name="container2") containers = self.client.containers.list(all=True) self.assertEqual(len(containers), 2) def test_start_container_with_random_port_bind(self): container = self.client.containers.create( image=constant.ALPINE, name="containerWithRandomBind", ports={"1234/tcp": None}, ) containers = self.client.containers.list(all=True) self.assertTrue(container in containers) def test_stop_container(self): top = self.client.containers.get(TestContainers.topContainerId) self.assertEqual(top.status, "running") # Stop a running container and validate the state top.stop() top.reload() self.assertIn(top.status, ("stopped", "exited")) def test_kill_container(self): top = self.client.containers.get(TestContainers.topContainerId) self.assertEqual(top.status, "running") # Kill a running container and validate the state top.kill() top.reload() self.assertIn(top.status, ("stopped", "exited")) def test_restart_container(self): # Validate the container state top = self.client.containers.get(TestContainers.topContainerId) top.stop() top.reload() self.assertIn(top.status, ("stopped", "exited")) # restart a running container and validate the state top.restart() top.reload() self.assertEqual(top.status, "running") def test_remove_container(self): # Remove container by ID with force top = self.client.containers.get(TestContainers.topContainerId) top.remove(force=True) self.assertEqual(len(self.client.containers.list()), 0) def test_remove_container_without_force(self): # Validate current container count self.assertEqual(len(self.client.containers.list()), 1) # Remove running container should throw error top = self.client.containers.get(TestContainers.topContainerId) with self.assertRaises(errors.APIError) as error: top.remove() self.assertEqual(error.exception.response.status_code, 500) # Remove container by ID without force top.stop() top.remove() self.assertEqual(len(self.client.containers.list()), 0) def test_pause_container(self): # Validate the container state top = self.client.containers.get(TestContainers.topContainerId) self.assertEqual(top.status, "running") # Pause a running container and validate the state top.pause() top.reload() self.assertEqual(top.status, "paused") def test_pause_stopped_container(self): # Stop the container top = self.client.containers.get(TestContainers.topContainerId) top.stop() # Pause exited container should throw error with self.assertRaises(errors.APIError) as error: top.pause() self.assertEqual(error.exception.response.status_code, 500) def test_unpause_container(self): top = self.client.containers.get(TestContainers.topContainerId) # Validate the container state top.pause() top.reload() self.assertEqual(top.status, "paused") # Pause a running container and validate the state top.unpause() top.reload() self.assertEqual(top.status, "running") def test_list_container(self): # Add container and validate the count self.client.containers.create(image="alpine", detach=True) containers = self.client.containers.list(all=True) self.assertEqual(len(containers), 2) def test_filters(self): self.skipTest("TODO Endpoint does not yet support filters") # List container with filter by id filters = {"id": TestContainers.topContainerId} ctnrs = self.client.containers.list(all=True, filters=filters) self.assertEqual(len(ctnrs), 1) # List container with filter by name filters = {"name": "top"} ctnrs = self.client.containers.list(all=True, filters=filters) self.assertEqual(len(ctnrs), 1) def test_copy_to_container(self): ctr: Optional[Container] = None try: test_file_content = b"Hello World!" ctr = self.client.containers.create(image="alpine", detach=True, command="top") ctr.start() buff: IO[bytes] = io.BytesIO() with tarfile.open(fileobj=buff, mode="w:") as tf: ti: tarfile.TarInfo = tarfile.TarInfo() ti.uid = 1042 ti.gid = 1043 ti.name = "a.txt" ti.path = "a.txt" ti.mode = 0o644 ti.type = tarfile.REGTYPE ti.size = len(test_file_content) tf.addfile(ti, fileobj=io.BytesIO(test_file_content)) buff.seek(0) ctr.put_archive("/tmp/", buff) ret, out = ctr.exec_run(["stat", "-c", "%u:%g", "/tmp/a.txt"]) self.assertEqual(ret, 0) self.assertEqual(out.rstrip(), b'1042:1043', "UID/GID of copied file") ret, out = ctr.exec_run(["cat", "/tmp/a.txt"]) self.assertEqual(ret, 0) self.assertEqual(out.rstrip(), test_file_content, "Content of copied file") finally: if ctr is not None: ctr.stop() ctr.remove() def test_mount_preexisting_dir(self): dockerfile = (B'FROM quay.io/libpod/alpine:latest\n' B'USER root\n' B'RUN mkdir -p /workspace\n' B'RUN chown 1042:1043 /workspace') img: Image img, out = self.client.images.build(fileobj=io.BytesIO(dockerfile)) ctr: Container = self.client.containers.create( image=img.id, detach=True, command="top", volumes=["test_mount_preexisting_dir_vol:/workspace"]) ctr.start() ret, out = ctr.exec_run(["stat", "-c", "%u:%g", "/workspace"]) self.assertEqual(out.rstrip(), b'1042:1043', "UID/GID set in dockerfile")
class TestContainers(unittest.TestCase): podman = None # initialized podman configuration for tests service = None # podman service instance topContainerId = "" def setUp(self): super().setUp() self.client = DockerClient(base_url="tcp://127.0.0.1:8080", timeout=15) TestContainers.podman.restore_image_from_cache(self.client) TestContainers.topContainerId = common.run_top_container(self.client) self.assertIsNotNone(TestContainers.topContainerId) def tearDown(self): common.remove_all_containers(self.client) common.remove_all_images(self.client) self.client.close() return super().tearDown() @classmethod def setUpClass(cls): super().setUpClass() TestContainers.podman = Podman() TestContainers.service = TestContainers.podman.open( "system", "service", "tcp:127.0.0.1:8080", "--time=0" ) # give the service some time to be ready... time.sleep(2) rc = TestContainers.service.poll() if rc is not None: raise subprocess.CalledProcessError(rc, "podman system service") @classmethod def tearDownClass(cls): TestContainers.service.terminate() stdout, stderr = TestContainers.service.communicate(timeout=0.5) if stdout: sys.stdout.write("\nContainers Service Stdout:\n" + stdout.decode("utf-8")) if stderr: sys.stderr.write("\nContainers Service Stderr:\n" + stderr.decode("utf-8")) TestContainers.podman.tear_down() return super().tearDownClass() def test_create_container(self): # Run a container with detach mode self.client.containers.create(image="alpine", detach=True) self.assertEqual(len(self.client.containers.list(all=True)), 2) def test_create_network(self): net = self.client.networks.create("testNetwork", driver="bridge") ctnr = self.client.containers.create(image="alpine", detach=True) # TODO fix when ready # This test will not work until all connect|disconnect # code is fixed. # net.connect(ctnr) # nets = self.client.networks.list(greedy=True) # self.assertGreaterEqual(len(nets), 1) # TODO fix endpoint to include containers # for n in nets: # if n.id == "testNetwork": # self.assertEqual(ctnr.id, n.containers) # self.assertTrue(False, "testNetwork not found") def test_start_container(self): # Podman docs says it should give a 304 but returns with no response # # Start a already started container should return 304 # response = self.client.api.start(container=TestContainers.topContainerId) # self.assertEqual(error.exception.response.status_code, 304) # Create a new container and validate the count self.client.containers.create(image=constant.ALPINE, name="container2") containers = self.client.containers.list(all=True) self.assertEqual(len(containers), 2) def test_stop_container(self): top = self.client.containers.get("top") self.assertEqual(top.status, "running") # Stop a running container and validate the state top.stop() top.reload() self.assertIn(top.status, ("stopped", "exited")) def test_restart_container(self): # Validate the container state top = self.client.containers.get(TestContainers.topContainerId) top.stop() top.reload() self.assertIn(top.status, ("stopped", "exited")) # restart a running container and validate the state top.restart() top.reload() self.assertEqual(top.status, "running") def test_remove_container(self): # Remove container by ID with force top = self.client.containers.get(TestContainers.topContainerId) top.remove(force=True) self.assertEqual(len(self.client.containers.list()), 0) def test_remove_container_without_force(self): # Validate current container count self.assertTrue(len(self.client.containers.list()), 1) # Remove running container should throw error top = self.client.containers.get(TestContainers.topContainerId) with self.assertRaises(errors.APIError) as error: top.remove() self.assertEqual(error.exception.response.status_code, 500) # Remove container by ID without force top.stop() top.remove() self.assertEqual(len(self.client.containers.list()), 0) def test_pause_container(self): # Validate the container state top = self.client.containers.get(TestContainers.topContainerId) self.assertEqual(top.status, "running") # Pause a running container and validate the state top.pause() top.reload() self.assertEqual(top.status, "paused") def test_pause_stopped_container(self): # Stop the container top = self.client.containers.get(TestContainers.topContainerId) top.stop() # Pause exited container should trow error with self.assertRaises(errors.APIError) as error: top.pause() self.assertEqual(error.exception.response.status_code, 500) def test_unpause_container(self): top = self.client.containers.get(TestContainers.topContainerId) # Validate the container state top.pause() top.reload() self.assertEqual(top.status, "paused") # Pause a running container and validate the state top.unpause() top.reload() self.assertEqual(top.status, "running") def test_list_container(self): # Add container and validate the count self.client.containers.create(image="alpine", detach=True) containers = self.client.containers.list(all=True) self.assertEqual(len(containers), 2) def test_filters(self): self.skipTest("TODO Endpoint does not yet support filters") # List container with filter by id filters = {"id": TestContainers.topContainerId} ctnrs = self.client.containers.list(all=True, filters=filters) self.assertEqual(len(ctnrs), 1) # List container with filter by name filters = {"name": "top"} ctnrs = self.client.containers.list(all=True, filters=filters) self.assertEqual(len(ctnrs), 1) def test_rename_container(self): top = self.client.containers.get(TestContainers.topContainerId) # rename bogus container with self.assertRaises(errors.APIError) as error: top.rename(name="newname") self.assertEqual(error.exception.response.status_code, 404)
class TestImages(unittest.TestCase): podman = None # initialized podman configuration for tests service = None # podman service instance def setUp(self): super().setUp() self.client = DockerClient(base_url="tcp://127.0.0.1:8080", timeout=15) TestImages.podman.restore_image_from_cache(self.client) def tearDown(self): common.remove_all_images(self.client) self.client.close() return super().tearDown() @classmethod def setUpClass(cls): super().setUpClass() TestImages.podman = Podman() TestImages.service = TestImages.podman.open( "system", "service", "tcp:127.0.0.1:8080", "--time=0" ) # give the service some time to be ready... time.sleep(2) returncode = TestImages.service.poll() if returncode is not None: raise subprocess.CalledProcessError(returncode, "podman system service") @classmethod def tearDownClass(cls): TestImages.service.terminate() stdout, stderr = TestImages.service.communicate(timeout=0.5) if stdout: sys.stdout.write("\nImages Service Stdout:\n" + stdout.decode("utf-8")) if stderr: sys.stderr.write("\nImAges Service Stderr:\n" + stderr.decode("utf-8")) TestImages.podman.tear_down() return super().tearDownClass() def test_tag_valid_image(self): """Validates if the image is tagged successfully""" alpine = self.client.images.get(constant.ALPINE) self.assertTrue(alpine.tag("demo", constant.ALPINE_SHORTNAME)) alpine = self.client.images.get(constant.ALPINE) for t in alpine.tags: self.assertIn("alpine", t) # @unittest.skip("doesn't work now") def test_retag_valid_image(self): """Validates if name updates when the image is retagged""" alpine = self.client.images.get(constant.ALPINE) self.assertTrue(alpine.tag("demo", "rename")) alpine = self.client.images.get(constant.ALPINE) self.assertNotIn("demo:test", alpine.tags) def test_list_images(self): """List images""" self.assertEqual(len(self.client.images.list()), 1) # Add more images self.client.images.pull(constant.BB) self.assertEqual(len(self.client.images.list()), 2) self.assertEqual(len(self.client.images.list(all=True)), 2) # List images with filter self.assertEqual(len(self.client.images.list(filters={"reference": "alpine"})), 1) def test_search_image(self): """Search for image""" for r in self.client.images.search("alpine"): # registry matches if string is in either one self.assertIn("alpine", r["Name"]+" "+r["Description"].lower()) def test_search_bogus_image(self): """Search for bogus image should throw exception""" try: r = self.client.images.search("bogus/bogus") except: return self.assertTrue(len(r) == 0) def test_remove_image(self): """Remove image""" # Check for error with wrong image name with self.assertRaises(errors.NotFound): self.client.images.remove("dummy") self.assertEqual(len(self.client.images.list()), 1) self.client.images.remove(constant.ALPINE) self.assertEqual(len(self.client.images.list()), 0) def test_image_history(self): """Image history""" img = self.client.images.get(constant.ALPINE) history = img.history() image_id = img.id[7:] if img.id.startswith("sha256:") else img.id found = False for change in history: found |= image_id in change.values() self.assertTrue(found, f"image id {image_id} not found in history") def test_get_image_exists_not(self): """Negative test for get image""" with self.assertRaises(errors.NotFound): response = self.client.images.get("image_does_not_exists") collections.deque(response) def test_save_image(self): """Export Image""" image = self.client.images.pull(constant.BB) file = os.path.join(TestImages.podman.image_cache, "busybox.tar") with open(file, mode="wb") as tarball: for frame in image.save(named=True): tarball.write(frame) sz = os.path.getsize(file) self.assertGreater(sz, 0) def test_load_image(self): """Import|Load Image""" self.assertEqual(len(self.client.images.list()), 1) image = self.client.images.pull(constant.BB) file = os.path.join(TestImages.podman.image_cache, "busybox.tar") with open(file, mode="wb") as tarball: for frame in image.save(): tarball.write(frame) with open(file, mode="rb") as saved: _ = self.client.images.load(saved) self.assertEqual(len(self.client.images.list()), 2) def test_load_corrupt_image(self): """Import|Load Image failure""" tarball = io.BytesIO("This is a corrupt tarball".encode("utf-8")) with self.assertRaises(APIError): self.client.images.load(tarball) def test_build_image(self): labels = {"apple": "red", "grape": "green"} _ = self.client.images.build( path="test/python/docker/build_labels", labels=labels, tag="labels", isolation="default" ) image = self.client.images.get("labels") self.assertEqual(image.labels["apple"], labels["apple"]) self.assertEqual(image.labels["grape"], labels["grape"])
#!/usr/bin/env python3 from docker import DockerClient import config docker_client = DockerClient(base_url=config.docker_base_url) docker_client.ping() docker_client.close() exit(0)
class DockerManager(): def __init__(self, docker_url=None, verbose=False): self._client = DockerClient(base_url=docker_url) self._docker_url = docker_url self._verbose = verbose try: self._client.ping() except APIError as err: logger.exception(err) raise DockerOperationError( 'Failed to connect to the Docker Engine.') def get_docker_info(self): """ Returns info about the local docker engine """ try: return { 'info': self._client.info(), 'version': self._client.version(), } except APIError as err: logger.exception(err) raise DockerOperationError( 'Failed to retrieve the Docker Engine info') def search_runnable_commands(self, image, tag, pull=True): """ Searching for ENTRYPOINT or CMD in a given image """ def _bind_shell(cmd_list): """ e.g. ["/bin/sh", "-c", "echo", "hello!"] """ if cmd_list[0] in SUPPORTED_SHELLS: return cmd_list else: # add a default shell on head return (DEFAULT_SHELL + cmd_list) full_name = '{0}:{1}'.format(image, tag) logger.info('Analyzing the [{}] image'.format(full_name)) image = self._pull_image_with_auth(image, tag) if image is not None: output = self._client.api.inspect_image(full_name) if not output or 'Config' not in output: raise FatalError( 'Failed to inspect {} image'.format(full_name)) else: entrypoint = output['Config']['Entrypoint'] cmd = output['Config']['Cmd'] commands = list() if entrypoint and not cmd: logger.debug( 'Detected only an ENTRYPOINT: {}'.format(entrypoint)) commands = _bind_shell(entrypoint) elif not entrypoint and cmd: logger.debug('Detected only a CMD: {}'.format(cmd)) commands = _bind_shell(cmd) elif entrypoint and cmd: logger.debug('Detected both ENTRYPOINT: \ {0} and CMD: {1}'.format(entrypoint, cmd)) # combining entrypoint (first) with cmd commands = _bind_shell(entrypoint) commands += cmd else: logger.info('the {} image is not runnable. \ Adding an infinite sleep'.format(full_name)) commands = DEFAULT_SHELL + ['sleep', 'infinity'] # add quotes # [0]: /bin/bash [1]: -c commands[2] = "\'" + commands[2] commands[-1] = commands[-1] + "\'" return ' '.join(commands) def _pull_image_with_auth(self, image, tag): try: return self._client.images.pull(image, tag) except ImageNotFound: # it should be an authentication error return self._image_authentication(image, tag) def _image_authentication(self, src_image, src_tag=None, auth=None): """ Handling Docker authentication if the image is hosted on a private repository Args: src_image (str): The original image that needs authentication for being fetched. src_tag (str): The tag of the image above. """ if src_tag is None: src_tag = 'latest' logger.warning('[{0}:{1}] image may not exist or authentication \ is required'.format(src_image, src_tag)) res = '' while res != 'YES': res = (input('[{0}:{1}] is correct? [Yes] Continue \ [No] Abort\n'.format(src_image, src_tag))).upper() if res == 'NO': logger.error( 'Docker image [{0}:{1}] cannot be found, the operation \ is aborted by the user.\n(Hint: Check the TOSCA manifest.)' .format(src_image, src_tag)) raise OperationAbortedByUser( CommonErrorMessages._DEFAULT_OPERATION_ABORTING_ERROR_MSG) attempts = 3 while attempts > 0: logger.info('Authenticate with the Docker Repository..') try: if auth is None: auth = create_auth_interactive( user_text='Enter the username: '******'Enter the password: '******'UNAUTHORIZED' or 'NOT FOUND' in msg: logger.info('Invalid username/password.') else: logger.exception(err) raise FatalError( CommonErrorMessages._DEFAULT_FATAL_ERROR_MSG) attempts -= 1 logger.info('Authentication failed. \ You have [{}] more attempts.'.format(attempts)) # Check authentication failure if attempts == 0: logger.error( 'You have used all the authentication attempts. Abort.') raise DockerAuthenticationFailedError( 'Authentication failed. Abort.') def _push_image(self, image, tag=None, auth=None, attempts=0): """ Push an image to a remote Docker Registry. Args: image (str): The name of the Docker Image to be pushed. tag (str): An optional tag for the Docker Image (default: 'latest') auth: An optional Dict for the authentication. attempts (int): The number of unsuccessful authentication attempts. """ if tag is None: tag = 'latest' if attempts == MAX_PUSH_ATTEMPTS: err = 'Reached max attempts for pushing \ [{0}] with tag [{1}]'.format(image, tag) logger.error(err) raise FatalError(err) logger.info('Pushing [{0}] with tag [{1}]'.format(image, tag)) result = self._client.images.push(image, tag=tag, auth_config=auth, stream=True, decode=True) # Access Denied detection for line in result: # an error is occurred if 'error' in line: # access denied if 'denied' or 'unauthorized' in line['error']: logger.info('Access to the repository denied. \ Authentication failed.') auth = create_auth_interactive() self._push_image(image, tag=tag, auth=auth, attempts=(attempts + 1)) else: logger.error( 'Unknown error during push of [{0}]: {1}'.format( image, line['error'])) raise FatalError( CommonErrorMessages._DEFAULT_FATAL_ERROR_MSG) logger.info('Image [{0}] with tag [{1}] successfully pushed.'.format( image, tag)) def _validate_build(self, build_logs): """ Validate the output of the building process. Args: build_logs (str): The output logs of a docker image building process. """ for log in build_logs: if 'stream' in log and self._verbose: print(log['stream']) if 'error' in log: logger.error('Error building the Docker Image:\ \n{}'.format(log['error'])) raise DockerOperationError('Failed to "toskosing" \ the Docker Image. Abort.') def _toskose_image_availability(self, toskose_image, toskose_tag='latest'): """ Check the availability of the official Docker Toskose image used during the "toskosing" process. Args: toskose_image (str): The official Toskose Docker image. toskose_tag (str): The tag of the image (default: 'latest'). """ try: _ = self._client.images.pull(toskose_image, tag=toskose_tag) except ImageNotFound: logger.error( 'Failed to retrieve the official Toskose image [{0}:{1}]. \ Abort'.format(toskose_image, toskose_tag)) raise DockerOperationError( 'The official Toskose image {0} not found. Try later.\ \nIf the problem persist, open an issue at {1}'.format( toskose_image, toskose_tag)) def _remove_previous_toskosed(self, image, tag=None): """ Remove previous toskosed images. Note: docker rmi doesn't remove an image if there are multiple tags referencing it. Args: image (str): The name of the Docker image. tag (str): The tag of the Docker Image (default: 'latest'). """ if tag is None: tag = 'latest' def print_well(tags): out = '' for tag in tags: out += '- {}\n'.format(tag) return out try: logger.info( 'Searching for previous toskosed images [{0}:{1}]'.format( image, tag)) image_found = self._client.images.get(image) logger.debug( 'Image [{0}] found. It\'s referenced by the following \ tags:\n\n{1}'.format(image, print_well(image_found.tags))) full_name = '{0}:{1}'.format(image, tag) if full_name in image_found.tags: self._client.images.remove(image=full_name, force=True) logger.info('Removed [{0}] reference from [{1}] image'.format( full_name, image)) try: image_found = self._client.images.get(image) logger.debug('Image [{0}][ID: {1}] still exist'.format( image, image_found.id)) except ImageNotFound: logger.info('Image [{0}][ID: {1}] doesn\'t have any \ references yet. Removed.'.format( image, image_found.id)) except ImageNotFound: logger.info('No previous image found.') def toskose_image(self, src_image, src_tag, dst_image, dst_tag, context, process_type, app_name, toskose_dockerfile=None, toskose_image=None, toskose_tag=None, enable_push=True): """ The process of "toskosing" the component(s) of a multi-component TOSCA-defined application. The "toskosing" process consists in merging contexts (e.g. artifacts, lifecycle scripts) of TOSCA software component(s) and a TOSCA container node in which they are hosted on in a new docker image. The TOSCA container node comes with a suggested docker image, which is "enriched" by the Toskose logic fetched remotely from the official Toskose Docker image, using a template Dockerfile by means of docker multi-stage features. This logic permits to handle the lifecycle of multiple components within the same container. (More details on how this process works can be found in the template Dockerfile in dockerfiles/ directory or in the official Toskose GitHub repository) In other words, this function consists in receiving a name of an existing docker image and a path to a build context, then by means of the docker client, the toskose template dockerfile is built and a new fresh docker image is generated. The new docker image includes the content of the original image, the contexts of the TOSCA software components and the logic of toskose for managing them. Args: src_image (str): The name of the image to be "toskosed". src_tag (str): The tag of the image to be "toskosed". dst_image (str): The name of the "toskosed" image. dst_tag (str): The tag of the "toskosed" image. context (str): The path of the application context. process_type (enum): The type of "toskosing" process. [unit/manager/free] app_name (str): The name of the TOSCA application. toskose_dockerfile (str): The path of the template dockerfile used in the process. toskose_image (str): The Docker Toskose base-image used in the "toskosing" process. toskose_tag (str): The tag of the Docker Toskose base-image. enable_push (bool): enable/disable pushing of the "toskosed" image. (default: True) """ if toskose_dockerfile is not None: if not os.path.exists(toskose_dockerfile): raise ValueError('The given toskose template dockerfile \ {} doesn\'t exist'.format(toskose_dockerfile)) if not app_name: raise ValueError('A name associated to the TOSCA application \ must be given') template_dir = os.path.join(os.path.dirname(__file__), DOCKERFILE_TEMPLATES_PATH) # TODO can be enanched if process_type == ToskosingProcessType.TOSKOSE_UNIT: if toskose_image is None: toskose_image = constants.DEFAULT_TOSKOSE_UNIT_BASE_IMAGE if toskose_tag is None: toskose_tag = constants.DEFAULT_TOSKOSE_UNIT_BASE_TAG if toskose_dockerfile is None: toskose_dockerfile = os.path.join( template_dir, DOCKERFILE_TOSKOSE_UNIT_TEMPLATE) elif process_type == ToskosingProcessType.TOSKOSE_MANAGER: if toskose_image is None: toskose_image = constants.DEFAULT_MANAGER_BASE_IMAGE if toskose_tag is None: toskose_tag = constants.DEFAULT_MANAGER_BASE_TAG if toskose_dockerfile is None: toskose_dockerfile = os.path.join( template_dir, DOCKERFILE_TOSKOSE_MANAGER_TEMPLATE) src_image = toskose_image src_tag = toskose_tag elif process_type == ToskosingProcessType.TOSKOSE_FREE: pass else: raise ValueError('Cannot recognize the "toskosing" process \ {}'.format(process_type)) if toskose_image is not None: self._toskose_image_availability(toskose_image, toskose_tag) # Check if the original image exists and needs authentication # to be fetched. # note: docker client does not distinguish between authentication # error or invalid image name (?) logger.info('Pulling [{0}:{1}]'.format(src_image, src_tag)) self._pull_image_with_auth(src_image, src_tag) # TODO can be removed? is it really necessary? self._remove_previous_toskosed(dst_image, dst_tag) logger.info('Toskosing [{0}:{1}] image'.format(src_image, src_tag)) try: build_args = { 'TOSCA_SRC_IMAGE': '{0}:{1}'.format(src_image, src_tag), 'TOSKOSE_BASE_IMG': '{0}:{1}'.format(toskose_image, toskose_tag) } image, build_logs = self._client.images.build( path=context, tag='{0}:{1}'.format(dst_image, dst_tag), buildargs=build_args, dockerfile=toskose_dockerfile, rm=True, # remove intermediate containers ) self._validate_build(build_logs) # push the "toskosed" image if enable_push: self._push_image(dst_image, tag=dst_tag) logger.info( '[{0}:{1}] image successfully toskosed in [{2}:{3}].'.format( src_image, src_tag, dst_image, dst_tag)) except (BuildError, APIError) as err: logger.exception(err) raise DockerOperationError('Failed to build image') def close(self): self._client.close()
from datetime import datetime, timezone from docker import DockerClient import dateutil.parser as parser from docker.models.containers import Container from requests.adapters import ReadTimeout from execute_cmd import execute_cmd from send_mail import notify_failure from utils import log, LogLevel import config # Test docker connection test_dc = DockerClient(base_url=config.docker_base_url) test_dc.ping() test_dc.close() def restart_container(container: Container): log(f"<27db178e> ({container.name}) Restaring container with timeout {config.container_stop_timeout}") container.restart(timeout=config.container_stop_timeout) def process_container(container: Container): log(f"<193643d5> ({container.name}) Container seems to be unhealthy") now = datetime.now(timezone.utc) failure_time = container.attrs["State"]["Health"]["Log"][-1]["End"] try: parsed_time = parser.isoparse(failure_time) if (now - parsed_time).seconds > config.container_debounce_time: execute_cmd(container)