def _deploy_config_pre_start(self, deploy_config): """Deploy a config before container is started. Most configs can be deployed before the container is up. For configs require a reboot to take effective, they must be deployed in this function. @param deploy_config: Config to be deployed. """ if not lxc_utils.path_exists(deploy_config.source): return # Path to the target file relative to host. if deploy_config.append: target = os.path.join(self.tmp_append, os.path.basename(deploy_config.target)) else: target = os.path.join(self.container.rootfs, deploy_config.target[1:]) # Recursively copy files/folder to the target. `-L` to always follow # symbolic links in source. target_dir = os.path.dirname(target) if not lxc_utils.path_exists(target_dir): utils.run('sudo mkdir -p "%s"' % target_dir) source = deploy_config.source # Make sure the source ends with `/.` if it's a directory. Otherwise # command cp will not work. if os.path.isdir(source) and source[-1] != '.': source += '/.' if source[-1] != '/' else '.' utils.run('sudo cp -RL "%s" "%s"' % (source, target))
def testPreStartWithCreate(self): """Verifies that pre-start creates mounted dirs. Checks that missing mount points are created when force_create is enabled. """ with TempDir() as tmpdir: src_dir = os.path.join(tmpdir, 'foobar') config = [{ 'mount': True, 'source': src_dir, 'target': '/target0', 'readonly': True, 'force_create': True }] with ConfigFile(config) as test_cfg, MockContainer() as container: manager = lxc_config.DeployConfigManager(container, test_cfg) # Pre-condition: the path doesn't exist. self.assertFalse(lxc_utils.path_exists(src_dir)) # After calling deploy_pre_start, the path should exist and the # mount should be created in the container. manager.deploy_pre_start() self.assertTrue(lxc_utils.path_exists(src_dir)) self.assertEqual(len(config), len(container.mounts)) for c in config: self.assertTrue(container.has_mount(c))
def _cleanup_host_mount(self): """Unmounts and removes the host dirs for this container.""" # Clean up all intermediate bind mounts into host_path and host_path_ro. for mount in self.mounts: mount.cleanup() # The SSP and other "real" content gets copied into the host dir. Use # rm -r to clear it out. if lxc_utils.path_exists(self.host_path): utils.run('sudo rm -r "%s"' % self.host_path) # The host_path_ro directory only contains intermediate bind points, # which should all have been cleared out. Use rmdir. if lxc_utils.path_exists(self.host_path_ro): utils.run('sudo rmdir "%s"' % self.host_path_ro)
def testHostDirExists(self): """Verifies that the host dir is just mounted if it already exists.""" # Pre-create the host dir and put a file in it. test_host_path = os.path.join(self.shared_host_path, 'testHostDirExists') test_filename = 'test_file' test_host_file = os.path.join(test_host_path, test_filename) test_string = 'jackdaws love my big sphinx of quartz.' os.mkdir(test_host_path) with open(test_host_file, 'w+') as f: f.write(test_string) # Sanity check self.assertTrue(lxc_utils.path_exists(test_host_file)) with self.createZygote(host_path=test_host_path) as zygote: zygote.start(wait_for_network=False) self.verifyBindMount(zygote, container_path=lxc.CONTAINER_AUTOTEST_DIR, host_path=zygote.host_path) # Verify that the old directory contents was preserved. cmd = 'cat %s' % os.path.join(lxc.CONTAINER_AUTOTEST_DIR, test_filename) test_output = zygote.attach_run(cmd).stdout.strip() self.assertEqual(test_string, test_output)
def __init__(self, container, config_file=None): """Initialize the deploy config manager. @param container: The container needs to deploy config. @param config_file: An optional config file. For testing. """ self.container = container # If shadow config is used, the deployment procedure will skip some # special handling of config file, e.g., # 1. Set enable_master_ssh to False in autotest shadow config. # 2. Set ssh logleve to ERROR for all hosts. if config_file is None: self.is_shadow_config = os.path.exists( SSP_DEPLOY_SHADOW_CONFIG_FILE) config_file = (SSP_DEPLOY_SHADOW_CONFIG_FILE if self.is_shadow_config else SSP_DEPLOY_CONFIG_FILE) else: self.is_shadow_config = False with open(config_file) as f: deploy_configs = json.load(f) self.deploy_configs = [ self.validate(c) for c in deploy_configs if 'append' in c ] self.mount_configs = [ self.validate_mount(c) for c in deploy_configs if 'mount' in c ] tmp_append = os.path.join(self.container.rootfs, _APPEND_FOLDER.lstrip(os.path.sep)) if lxc_utils.path_exists(tmp_append): utils.run('sudo rm -rf "%s"' % tmp_append) utils.run('sudo mkdir -p "%s"' % tmp_append)
def clone(cls, src, new_name, new_path=None, snapshot=False, cleanup=False): """Creates a clone of this container. @param src: The original container. @param new_name: Name for the cloned container. @param new_path: LXC path for the cloned container (optional; if not specified, the new container is created in the same directory as the source container). @param snapshot: Whether to snapshot, or create a full clone. @param cleanup: If a container with the given name and path already exist, clean it up first. """ if new_path is None: new_path = src.container_path # If a container exists at this location, clean it up first container_folder = os.path.join(new_path, new_name) if lxc_utils.path_exists(container_folder): if not cleanup: raise error.ContainerError('Container %s already exists.' % new_name) container = Container.createFromExistingDir(new_path, new_name) try: container.destroy() except error.CmdError as e: # The container could be created in a incompleted state. Delete # the container folder instead. logging.warn('Failed to destroy container %s, error: %s', new_name, e) utils.run('sudo rm -rf "%s"' % container_folder) # Create and return the new container. return cls(new_path, new_name, {}, src, snapshot)
def testCleanup_withClones(self): """Verifies that cleanup cleans up the base image. Ensure that it works even when clones of the base image exist. """ # Do not snapshot, as snapshots of snapshots behave differently than # snapshots of full container clones. BaseImage cleanup code assumes # that the base container is not a snapshot. base = lxc.Container.clone(src=reference_container, new_name=constants.BASE, new_path=test_dir, snapshot=False) manager = BaseImage(base.container_path, base.name) clones = [] for i in range(3): clones.append( lxc.Container.clone(src=base, new_name='clone_%d' % i, snapshot=True)) # Precondition: all containers are valid. base.refresh_status() for container in clones: container.refresh_status() manager.cleanup() # Verify that all containers were cleaned up self.assertFalse( lxc_utils.path_exists(os.path.join(base.container_path, base.name))) for container in clones: if constants.SUPPORT_SNAPSHOT_CLONE: # Snapshot clones should get deleted along with the base # container. self.assertFalse( lxc_utils.path_exists( os.path.join(container.container_path, container.name))) else: # If snapshot clones aren't supported (e.g. on moblab), the # clones should not be affected by the destruction of the base # container. try: container.refresh_status() except error.ContainerError: self.fail(error.format_error())
def setup(self, name=None, force_delete=False): """Download and setup the base container. @param name: Name of the base container, defaults to the name passed to the constructor. If a different name is provided, that name overrides the name originally passed to the constructor. @param force_delete: True to force to delete existing base container. This action will destroy all running test containers. Default is set to False. """ if name is not None: self.base_name = name if not self.container_path: raise error.ContainerError( 'You must set a valid directory to store containers in ' 'global config "AUTOSERV/ container_path".') if not os.path.exists(self.container_path): os.makedirs(self.container_path) if self.base_container and not force_delete: logging.error( 'Base container already exists. Set force_delete to True ' 'to force to re-stage base container. Note that this ' 'action will destroy all running test containers') # Set proper file permission. base container in moblab may have # owner of not being root. Force to update the folder's owner. self._set_root_owner() return # Destroy existing base container if exists. if self.base_container: self.cleanup() try: self._download_and_install_base_container() self._set_root_owner() except: # Clean up if something went wrong. base_path = os.path.join(self.container_path, self.base_name) if lxc_utils.path_exists(base_path): exc_info = sys.exc_info() container = Container.create_from_existing_dir( self.container_path, self.base_name) # Attempt destroy. Log but otherwise ignore errors. try: container.destroy() except error.CmdError as e: logging.error(e) # Raise the cached exception with original backtrace. raise exc_info[0], exc_info[1], exc_info[2] else: raise else: self.base_container = Container.create_from_existing_dir( self.container_path, self.base_name)
def clone(cls, src, new_name=None, new_path=None, snapshot=False, cleanup=False): """Creates a clone of this container. @param src: The original container. @param new_name: Name for the cloned container. If this is not provided, a random unique container name will be generated. @param new_path: LXC path for the cloned container (optional; if not specified, the new container is created in the same directory as the source container). @param snapshot: Whether to snapshot, or create a full clone. Note that snapshot cloning is not supported on all platforms. If this code is running on a platform that does not support snapshot clones, this flag is ignored. @param cleanup: If a container with the given name and path already exist, clean it up first. """ if new_path is None: new_path = src.container_path if new_name is None: _, new_name = os.path.split( tempfile.mkdtemp(dir=new_path, prefix='container.')) logging.debug('Generating new name for container: %s', new_name) else: # If a container exists at this location, clean it up first container_folder = os.path.join(new_path, new_name) if lxc_utils.path_exists(container_folder): if not cleanup: raise error.ContainerError('Container %s already exists.' % new_name) container = Container.create_from_existing_dir( new_path, new_name) try: container.destroy() except error.CmdError as e: # The container could be created in a incompleted # state. Delete the container folder instead. logging.warn('Failed to destroy container %s, error: %s', new_name, e) utils.run('sudo rm -rf "%s"' % container_folder) # Create the directory prior to creating the new container. This # puts the ownership of the container under the current process's # user, rather than root. This is necessary to enable the # ContainerId to serialize properly. os.mkdir(container_folder) # Create and return the new container. new_container = cls(new_path, new_name, {}, src, snapshot) return new_container
def _setup_shared_host_path(self): """Sets up the shared host directory.""" # First, clear out the old shared host dir if it exists. if lxc_utils.path_exists(self.shared_host_path): self._cleanup_shared_host_path() # Create the dir and set it up as a shared mount point. utils.run(('sudo mkdir "{path}" && ' 'sudo mount --bind "{path}" "{path}" && ' 'sudo mount --make-unbindable "{path}" && ' 'sudo mount --make-shared "{path}"') .format(path=self.shared_host_path))
def _find_clones(self): """Finds snapshot clones of the current base container.""" snapshot_file = os.path.join(self.container_path, self.base_name, 'lxc_snapshots') if not lxc_utils.path_exists(snapshot_file): return cmd = 'sudo cat %s' % snapshot_file clone_info = [ line.strip() for line in utils.run(cmd).stdout.splitlines() ] # lxc_snapshots contains pairs of lines (lxc_path, container_name). for i in range(0, len(clone_info), 2): lxc_path = clone_info[i] name = clone_info[i + 1] yield Container.create_from_existing_dir(lxc_path, name)
def main(): """Clean up the remnants from any old aborted unit tests.""" # Manually clean out the host dir. if lxc_utils.path_exists(TEST_HOST_PATH): for host_dir in os.listdir(TEST_HOST_PATH): host_dir = os.path.realpath(os.path.join(TEST_HOST_PATH, host_dir)) try: utils.run('sudo umount %s' % host_dir) except error.CmdError: pass utils.run('sudo rm -r %s' % host_dir) # Utilize the container_bucket to clear out old test containers. bucket = lxc.ContainerBucket(TEST_CONTAINER_PATH, TEST_HOST_PATH) bucket.destroy_all()
def __init__(self, lxc_path, host_path): self.fast_setup = False try: if lxc_utils.path_exists( os.path.join(constants.DEFAULT_CONTAINER_PATH, constants.BASE)): lxc_path = os.path.realpath(lxc_path) if not lxc_utils.path_exists(lxc_path): os.makedirs(lxc_path) # Clone the base container (snapshot for speed) to make a base # container for the unit test. base = lxc.Container.createFromExistingDir( constants.DEFAULT_CONTAINER_PATH, constants.BASE) lxc.Container.clone(src=base, new_name=constants.BASE, new_path=lxc_path, snapshot=True, cleanup=False) self.fast_setup = True finally: super(FastContainerBucket, self).__init__(lxc_path, host_path) if self.base_container is not None: self._setup_shared_host_path()
def _get_container_info_moblab(container_path, **filters): """Get a collection of container information in the given container path in a Moblab. TODO(crbug.com/457496): remove this method once python 3 can be installed in Moblab and lxc-ls command can use python 3 code. When running in Moblab, lxc-ls behaves differently from a server with python 3 installed: 1. lxc-ls returns a list of containers installed under /etc/lxc, the default lxc container directory. 2. lxc-ls --active lists all active containers, regardless where the container is located. For such differences, we have to special case Moblab to make the behavior close to a server with python 3 installed. That is, 1. List only containers in a given folder. 2. Assume all active containers have state of RUNNING. @param container_path: Path to look for containers. @param filters: Key value to filter the containers, e.g., name='base' @return: A list of dictionaries that each dictionary has the information of a container. The keys are defined in ATTRIBUTES. """ info_collection = [] active_containers = common_utils.run('sudo lxc-ls --active').stdout.split() name_filter = filters.get('name', None) state_filter = filters.get('state', None) if filters and set(filters.keys()) - set(['name', 'state']): raise error.ContainerError('When running in Moblab, container list ' 'filter only supports name and state.') for name in os.listdir(container_path): # Skip all files and folders without rootfs subfolder. if (os.path.isfile(os.path.join(container_path, name)) or not lxc_utils.path_exists( os.path.join(container_path, name, 'rootfs'))): continue info = { 'name': name, 'state': 'RUNNING' if name in active_containers else 'STOPPED' } if ((name_filter and name_filter != info['name']) or (state_filter and state_filter != info['state'])): continue info_collection.append(info) return info_collection
def testCleanup_noClones(self): """Verifies that cleanup cleans up the base image.""" base = lxc.Container.clone(src=reference_container, new_name=constants.BASE, new_path=test_dir, snapshot=True) manager = BaseImage(base.container_path, base.name) # Precondition: ensure base exists and is a valid container. base.refresh_status() manager.cleanup() # Verify that the base container was cleaned up. self.assertFalse(lxc_utils.path_exists( os.path.join(base.container_path, base.name)))
def _deploy_config_pre_start(self, deploy_config): """Deploy a config before container is started. Most configs can be deployed before the container is up. For configs require a reboot to take effective, they must be deployed in this function. @param deploy_config: Config to be deployed. """ if not lxc_utils.path_exists(deploy_config.source): return # Path to the target file relative to host. if deploy_config.append: target = os.path.join(_APPEND_FOLDER, os.path.basename(deploy_config.target)) else: target = deploy_config.target self.container.copy(deploy_config.source, target)
def __init__(self, container_path, name, attribute_values, src=None, snapshot=False, host_path=None): """Initialize an object of LXC container with given attribute values. @param container_path: Directory that stores the container. @param name: Name of the container. @param attribute_values: A dictionary of attribute values for the container. @param src: An optional source container. If provided, the source continer is cloned, and the new container will point to the clone. @param snapshot: Whether or not to create a snapshot clone. By default, this is false. If a snapshot is requested and creating a snapshot clone fails, a full clone will be attempted. @param host_path: If set to None (the default), a host path will be generated based on constants.DEFAULT_SHARED_HOST_PATH. Otherwise, this can be used to override the host path of the new container, for testing purposes. """ super(Zygote, self).__init__(container_path, name, attribute_values, src, snapshot) # Initialize host dir and mount if host_path is None: self.host_path = os.path.join( os.path.realpath(constants.DEFAULT_SHARED_HOST_PATH), self.name) else: self.host_path = host_path if src is not None: # If creating a new zygote, initialize the host dir. if not lxc_utils.path_exists(self.host_path): utils.run('sudo mkdir %s' % self.host_path) self.mount_dir(self.host_path, constants.CONTAINER_AUTOTEST_DIR)
def destroy(self, force=True): super(Zygote, self).destroy(force) if lxc_utils.path_exists(self.host_path): self._cleanup_host_mount()
def __init__(self, container_path, name, attribute_values, src=None, snapshot=False, host_path=None): """Initialize an object of LXC container with given attribute values. @param container_path: Directory that stores the container. @param name: Name of the container. @param attribute_values: A dictionary of attribute values for the container. @param src: An optional source container. If provided, the source continer is cloned, and the new container will point to the clone. @param snapshot: Whether or not to create a snapshot clone. By default, this is false. If a snapshot is requested and creating a snapshot clone fails, a full clone will be attempted. @param host_path: If set to None (the default), a host path will be generated based on constants.DEFAULT_SHARED_HOST_PATH. Otherwise, this can be used to override the host path of the new container, for testing purposes. """ # Check if this is a pre-existing LXC container. Do this before calling # the super ctor, because that triggers container creation. exists = lxc.get_container_info(container_path, name=name) super(Zygote, self).__init__(container_path, name, attribute_values, src, snapshot) logging.debug('Creating Zygote (lxcpath:%s name:%s)', container_path, name) # host_path is a directory within a shared bind-mount, which enables # bind-mounts from the host system to be shared with the LXC container. if host_path is not None: # Allow the host_path to be injected, for testing. self.host_path = host_path else: if exists: # Pre-existing Zygotes must have a host path. self.host_path = self._find_existing_host_dir() if self.host_path is None: raise error.ContainerError( 'Container %s has no host path.' % os.path.join(container_path, name)) else: # New Zygotes use a predefined template to generate a host path. self.host_path = os.path.join( os.path.realpath(constants.DEFAULT_SHARED_HOST_PATH), self.name) # host_path_ro is a directory for holding intermediate mount points, # which are necessary when creating read-only bind mounts. See the # mount_dir method for more details. # # Generate a host_path_ro based on host_path. ro_dir, ro_name = os.path.split(self.host_path.rstrip(os.path.sep)) self.host_path_ro = os.path.join(ro_dir, '%s.ro' % ro_name) # Remember mounts so they can be cleaned up in destroy. self.mounts = [] if exists: self._find_existing_bind_mounts() else: # Creating a new Zygote - initialize the host dirs. Don't use sudo, # so that the resulting directories can be accessed by autoserv (for # SSP installation, etc). if not lxc_utils.path_exists(self.host_path): os.makedirs(self.host_path) if not lxc_utils.path_exists(self.host_path_ro): os.makedirs(self.host_path_ro) # Create the mount point within the container's rootfs. # Changes within container's rootfs require sudo. utils.run('sudo mkdir %s' % os.path.join( self.rootfs, constants.CONTAINER_HOST_DIR.lstrip(os.path.sep))) self.mount_dir(self.host_path, constants.CONTAINER_HOST_DIR)