def verify_autotest_setup(self, job_id):
"""Verify autotest code is set up properly in the container.
@param job_id: ID of the job, used to format job result folder.
@raise ContainerError: If autotest code is not set up properly.
"""
# Test autotest code is setup by verifying a list of
# (directory, minimum file count)
if IS_MOBLAB:
site_packages_path = MOBLAB_SITE_PACKAGES_CONTAINER
else:
site_packages_path = os.path.join(lxc_config.CONTAINER_AUTOTEST_DIR,
'site-packages')
directories_to_check = [
(lxc_config.CONTAINER_AUTOTEST_DIR, 3),
(RESULT_DIR_FMT % job_id, 0),
(site_packages_path, 3)]
for directory, count in directories_to_check:
result = self.attach_run(command=(COUNT_FILE_CMD %
{'dir': directory})).stdout
logging.debug('%s entries in %s.', int(result), directory)
if int(result) < count:
raise error.ContainerError('%s is not properly set up.' %
directory)
def start(self, wait_for_network=True):
"""Start the container.
@param wait_for_network: True to wait for network to be up. Default is
set to True.
@raise ContainerError: If container does not exist, or fails to start.
"""
cmd = 'sudo lxc-start -P %s -n %s -d' % (self.container_path,
self.name)
output = utils.run(cmd).stdout
if not self.is_running():
raise error.ContainerError(
'Container %s failed to start. lxc command output:\n%s' %
(os.path.join(self.container_path, self.name), output))
if wait_for_network:
logging.debug('Wait for network to be up.')
start_time = time.time()
utils.poll_for_condition(
condition=self.is_network_up,
timeout=constants.NETWORK_INIT_TIMEOUT,
sleep_interval=constants.NETWORK_INIT_CHECK_INTERVAL)
logging.debug('Network is up after %.2f seconds.',
time.time() - start_time)
def install_package_precheck(packages):
"""If SSP is not enabled or the test is running in chroot (using test_that),
packages installation should be skipped.
The check does not raise exception so tests started by test_that or running
in an Autotest setup with SSP disabled can continue. That assume the running
environment, chroot or a machine, has the desired packages installed
already.
@param packages: A list of names of the packages to install.
@return: True if package installation can continue. False if it should be
skipped.
"""
if not SSP_ENABLED and not utils.is_in_container():
logging.info(
'Server-side packaging is not enabled. Install package %s '
'is skipped.', packages)
return False
if server_utils.is_inside_chroot():
logging.info(
'Test is running inside chroot. Install package %s is '
'skipped.', packages)
return False
if not utils.is_in_container():
raise error.ContainerError('Package installation is only supported '
'when test is running inside container.')
return True
def clone(cls, src, new_name, new_path=None, snapshot=False, cleanup=False):
"""Creates a clone of this container.
@param src: The original container.
@param new_name: Name for the cloned container.
@param new_path: LXC path for the cloned container (optional; if not
specified, the new container is created in the same directory as
the source container).
@param snapshot: Whether to snapshot, or create a full clone.
@param cleanup: If a container with the given name and path already
exist, clean it up first.
"""
if new_path is None:
new_path = src.container_path
# If a container exists at this location, clean it up first
container_folder = os.path.join(new_path, new_name)
if lxc_utils.path_exists(container_folder):
if not cleanup:
raise error.ContainerError('Container %s already exists.' %
new_name)
container = Container.createFromExistingDir(new_path, new_name)
try:
container.destroy()
except error.CmdError as e:
# The container could be created in a incompleted state. Delete
# the container folder instead.
logging.warn('Failed to destroy container %s, error: %s',
new_name, e)
utils.run('sudo rm -rf "%s"' % container_folder)
# Create and return the new container.
return cls(new_path, new_name, {}, src, snapshot)
def verify_autotest_setup(self, job_folder):
"""Verify autotest code is set up properly in the container.
@param job_folder: Name of the job result folder.
@raise ContainerError: If autotest code is not set up properly.
"""
# Test autotest code is setup by verifying a list of
# (directory, minimum file count)
directories_to_check = [(constants.CONTAINER_AUTOTEST_DIR, 3),
(constants.RESULT_DIR_FMT % job_folder, 0),
(constants.CONTAINER_SITE_PACKAGES_PATH, 3)]
for directory, count in directories_to_check:
result = self.attach_run(command=(constants.COUNT_FILE_CMD % {
'dir': directory
})).stdout
logging.debug('%s entries in %s.', int(result), directory)
if int(result) < count:
raise error.ContainerError('%s is not properly set up.' %
directory)
# lxc-attach and run command does not run in shell, thus .bashrc is not
# loaded. Following command creates a symlink in /usr/bin/ for gsutil
# if it's installed.
# TODO(dshi): Remove this code after lab container is updated with
# gsutil installed in /usr/bin/
self.attach_run('test -f /root/gsutil/gsutil && '
'ln -s /root/gsutil/gsutil /usr/bin/gsutil || true')
def setup(self, name=None, force_delete=False):
"""Download and setup the base container.
@param name: Name of the base container, defaults to the name passed to
the constructor. If a different name is provided, that
name overrides the name originally passed to the
constructor.
@param force_delete: True to force to delete existing base container.
This action will destroy all running test
containers. Default is set to False.
"""
if name is not None:
self.base_name = name
if not self.container_path:
raise error.ContainerError(
'You must set a valid directory to store containers in '
'global config "AUTOSERV/ container_path".')
if not os.path.exists(self.container_path):
os.makedirs(self.container_path)
if self.base_container and not force_delete:
logging.error(
'Base container already exists. Set force_delete to True '
'to force to re-stage base container. Note that this '
'action will destroy all running test containers')
# Set proper file permission. base container in moblab may have
# owner of not being root. Force to update the folder's owner.
self._set_root_owner()
return
# Destroy existing base container if exists.
if self.base_container:
self.cleanup()
try:
self._download_and_install_base_container()
self._set_root_owner()
except:
# Clean up if something went wrong.
base_path = os.path.join(self.container_path, self.base_name)
if lxc_utils.path_exists(base_path):
exc_info = sys.exc_info()
container = Container.create_from_existing_dir(
self.container_path, self.base_name)
# Attempt destroy. Log but otherwise ignore errors.
try:
container.destroy()
except error.CmdError as e:
logging.error(e)
# Raise the cached exception with original backtrace.
raise exc_info[0], exc_info[1], exc_info[2]
else:
raise
else:
self.base_container = Container.create_from_existing_dir(
self.container_path, self.base_name)
def create_from_base(self, name, disable_snapshot_clone=False,
force_cleanup=False):
"""Create a container from the base container.
@param name: Name of the container.
@param disable_snapshot_clone: Set to True to force to clone without
using snapshot clone even if the host supports that.
@param force_cleanup: Force to cleanup existing container.
@return: A Container object for the created container.
@raise ContainerError: If the container already exist.
@raise error.CmdError: If lxc-clone call failed for any reason.
"""
if self.exist(name) and not force_cleanup:
raise error.ContainerError('Container %s already exists.' % name)
# Cleanup existing container with the given name.
container_folder = os.path.join(self.container_path, name)
if lxc_utils.path_exists(container_folder) and force_cleanup:
container = Container(self.container_path, {'name': name})
try:
container.destroy()
except error.CmdError as e:
# The container could be created in a incompleted state. Delete
# the container folder instead.
logging.warn('Failed to destroy container %s, error: %s',
name, e)
utils.run('sudo rm -rf "%s"' % container_folder)
use_snapshot = SUPPORT_SNAPSHOT_CLONE and not disable_snapshot_clone
snapshot = '-s' if use_snapshot else ''
# overlayfs is the default clone backend storage. However it is not
# supported in Ganeti yet. Use aufs as the alternative.
aufs = '-B aufs' if utils.is_vm() and use_snapshot else ''
cmd = ('sudo lxc-clone -p %s -P %s %s' %
(self.container_path, self.container_path,
' '.join([BASE, name, snapshot, aufs])))
try:
utils.run(cmd)
return self.get(name)
except error.CmdError:
if not use_snapshot:
raise
else:
# Snapshot clone failed, retry clone without snapshot. The retry
# won't hit the code here and cause an infinite loop as
# disable_snapshot_clone is set to True.
container = self.create_from_base(
name, disable_snapshot_clone=True, force_cleanup=True)
# Report metadata about retry success.
autotest_es.post(use_http=True,
type_str=CONTAINER_CREATE_RETRY_METADB_TYPE,
metadata={'drone': socket.gethostname(),
'name': name,
'success': True})
return container
def clone(cls,
src,
new_name=None,
new_path=None,
snapshot=False,
cleanup=False):
"""Creates a clone of this container.
@param src: The original container.
@param new_name: Name for the cloned container. If this is not
provided, a random unique container name will be
generated.
@param new_path: LXC path for the cloned container (optional; if not
specified, the new container is created in the same
directory as the source container).
@param snapshot: Whether to snapshot, or create a full clone. Note that
snapshot cloning is not supported on all platforms. If
this code is running on a platform that does not
support snapshot clones, this flag is ignored.
@param cleanup: If a container with the given name and path already
exist, clean it up first.
"""
if new_path is None:
new_path = src.container_path
if new_name is None:
_, new_name = os.path.split(
tempfile.mkdtemp(dir=new_path, prefix='container.'))
logging.debug('Generating new name for container: %s', new_name)
else:
# If a container exists at this location, clean it up first
container_folder = os.path.join(new_path, new_name)
if lxc_utils.path_exists(container_folder):
if not cleanup:
raise error.ContainerError('Container %s already exists.' %
new_name)
container = Container.create_from_existing_dir(
new_path, new_name)
try:
container.destroy()
except error.CmdError as e:
# The container could be created in a incompleted
# state. Delete the container folder instead.
logging.warn('Failed to destroy container %s, error: %s',
new_name, e)
utils.run('sudo rm -rf "%s"' % container_folder)
# Create the directory prior to creating the new container. This
# puts the ownership of the container under the current process's
# user, rather than root. This is necessary to enable the
# ContainerId to serialize properly.
os.mkdir(container_folder)
# Create and return the new container.
new_container = cls(new_path, new_name, {}, src, snapshot)
return new_container
def setup_base(self, name=BASE, force_delete=False):
"""Setup base container.
@param name: Name of the base container, default to base.
@param force_delete: True to force to delete existing base container.
This action will destroy all running test
containers. Default is set to False.
"""
if not self.container_path:
raise error.ContainerError(
'You must set a valid directory to store containers in '
'global config "AUTOSERV/ container_path".')
if not os.path.exists(self.container_path):
os.makedirs(self.container_path)
base_path = os.path.join(self.container_path, name)
if self.exist(name) and not force_delete:
logging.error(
'Base container already exists. Set force_delete to True '
'to force to re-stage base container. Note that this '
'action will destroy all running test containers')
# Set proper file permission. base container in moblab may have
# owner of not being root. Force to update the folder's owner.
# TODO(dshi): Change root to current user when test container can be
# unprivileged container.
utils.run('sudo chown -R root "%s"' % base_path)
utils.run('sudo chgrp -R root "%s"' % base_path)
return
# Destroy existing base container if exists.
if self.exist(name):
# TODO: We may need to destroy all snapshots created from this base
# container, not all container.
self.destroy_all()
# Download and untar the base container.
tar_path = os.path.join(self.container_path, '%s.tar.xz' % name)
path_to_cleanup = [tar_path, base_path]
for path in path_to_cleanup:
if os.path.exists(path):
utils.run('sudo rm -rf "%s"' % path)
container_url = CONTAINER_BASE_URL_FMT % name
download_extract(container_url, tar_path, self.container_path)
# Remove the downloaded container tar file.
utils.run('sudo rm "%s"' % tar_path)
# Set proper file permission.
# TODO(dshi): Change root to current user when test container can be
# unprivileged container.
utils.run('sudo chown -R root "%s"' % base_path)
utils.run('sudo chgrp -R root "%s"' % base_path)
# Update container config with container_path from global config.
config_path = os.path.join(base_path, 'config')
utils.run('sudo sed -i "s|container_dir|%s|g" "%s"' %
(self.container_path, config_path))
def refresh_status(self):
"""Refresh the status information of the container.
"""
containers = get_container_info(self.container_path, name=self.name)
if not containers:
raise error.ContainerError(
'No container found in directory %s with name of %s.' %
self.container_path, self.name)
attribute_values = containers[0]
for attribute, value in attribute_values.iteritems():
setattr(self, attribute, value)
def stop(self):
"""Stop the container.
@raise ContainerError: If container does not exist, or fails to start.
"""
cmd = 'sudo lxc-stop -P %s -n %s' % (self.container_path, self.name)
output = utils.run(cmd).stdout
self.refresh_status()
if self.state != 'STOPPED':
raise error.ContainerError(
'Container %s failed to be stopped. lxc command output:\n'
'%s' % (os.path.join(self.container_path, self.name), output))
def _get_container_info_moblab(container_path, **filters):
"""Get a collection of container information in the given container path
in a Moblab.
TODO(crbug.com/457496): remove this method once python 3 can be installed
in Moblab and lxc-ls command can use python 3 code.
When running in Moblab, lxc-ls behaves differently from a server with python
3 installed:
1. lxc-ls returns a list of containers installed under /etc/lxc, the default
lxc container directory.
2. lxc-ls --active lists all active containers, regardless where the
container is located.
For such differences, we have to special case Moblab to make the behavior
close to a server with python 3 installed. That is,
1. List only containers in a given folder.
2. Assume all active containers have state of RUNNING.
@param container_path: Path to look for containers.
@param filters: Key value to filter the containers, e.g., name='base'
@return: A list of dictionaries that each dictionary has the information of
a container. The keys are defined in ATTRIBUTES.
"""
info_collection = []
active_containers = utils.run('sudo lxc-ls --active').stdout.split()
name_filter = filters.get('name', None)
state_filter = filters.get('state', None)
if filters and set(filters.keys()) - set(['name', 'state']):
raise error.ContainerError('When running in Moblab, container list '
'filter only supports name and state.')
for name in os.listdir(container_path):
# Skip all files and folders without rootfs subfolder.
if (os.path.isfile(os.path.join(container_path, name))
or not lxc_utils.path_exists(
os.path.join(container_path, name, 'rootfs'))):
continue
info = {
'name': name,
'state': 'RUNNING' if name in active_containers else 'STOPPED'
}
if ((name_filter and name_filter != info['name'])
or (state_filter and state_filter != info['state'])):
continue
info_collection.append(info)
return info_collection
def _get_lxc_config(self, key):
"""Retrieves an LXC config value from the container.
@param key The key of the config value to retrieve.
"""
cmd = ('sudo lxc-info -P %s -n %s -c %s' %
(self.container_path, self.name, key))
config = utils.run(cmd).stdout.strip().splitlines()
# Strip the decoration from line 1 of the output.
match = re.match('%s = (.*)' % key, config[0])
if not match:
raise error.ContainerError(
'Config %s not found for container %s. (%s)' %
(key, self.name, ','.join(config)))
config[0] = match.group(1)
return config
def set_hostname(self, hostname):
"""Sets the hostname within the container.
This method can only be called on a running container.
@param hostname The new container hostname.
@raise ContainerError: If the container is not running.
"""
if not self.is_running():
raise error.ContainerError(
'set_hostname can only be called on running containers.')
self.attach_run('hostname %s' % (hostname))
self.attach_run(constants.APPEND_CMD_FMT % {
'content': '127.0.0.1 %s' % (hostname),
'file': '/etc/hosts'
})
def rootfs(self):
"""Path to the rootfs of the container.
This property returns the path to the rootfs of the container, that is,
the folder where the container stores its local files. It reads the
attribute lxc.rootfs from the config file of the container, e.g.,
lxc.rootfs = /usr/local/autotest/containers/t4/rootfs
If the container is created with snapshot, the rootfs is a chain of
folders, separated by `:` and ordered by how the snapshot is created,
e.g.,
lxc.rootfs = overlayfs:/usr/local/autotest/containers/base/rootfs:
/usr/local/autotest/containers/t4_s/delta0
This function returns the last folder in the chain, in above example,
that is `/usr/local/autotest/containers/t4_s/delta0`
Files in the rootfs will be accessible directly within container. For
example, a folder in host "[rootfs]/usr/local/file1", can be accessed
inside container by path "/usr/local/file1". Note that symlink in the
host can not across host/container boundary, instead, directory mount
should be used, refer to function mount_dir.
@return: Path to the rootfs of the container.
"""
if not self._rootfs:
cmd = ('sudo lxc-info -P %s -n %s -c lxc.rootfs' %
(self.container_path, self.name))
lxc_rootfs_config = utils.run(cmd).stdout.strip()
match = re.match('lxc.rootfs = (.*)', lxc_rootfs_config)
if not match:
raise error.ContainerError(
'Failed to locate rootfs for container %s. lxc.rootfs '
'in the container config file is %s' %
(self.name, lxc_rootfs_config))
lxc_rootfs = match.group(1)
self.clone_from_snapshot = ':' in lxc_rootfs
if self.clone_from_snapshot:
self._rootfs = lxc_rootfs.split(':')[-1]
else:
self._rootfs = lxc_rootfs
return self._rootfs
def install_packages(packages=[], python_packages=[]):
"""Install the given package inside container.
@param packages: A list of names of the packages to install.
@param python_packages: A list of names of the python packages to install
using pip.
@raise error.ContainerError: If package is attempted to be installed outside
a container.
@raise error.CmdError: If the package doesn't exist or failed to install.
"""
if not install_package_precheck(packages or python_packages):
return
if not utils.is_in_container():
raise error.ContainerError('Package installation is only supported '
'when test is running inside container.')
# Always run apt-get update before installing any container. The base
# container may have outdated cache.
utils.run('sudo apt-get update')
# Make sure the lists are not None for iteration.
packages = [] if not packages else packages
if python_packages:
packages.extend(['python-pip', 'python-dev'])
if packages:
utils.run('sudo apt-get install %s -y --force-yes' % ' '.join(packages))
logging.debug('Packages are installed: %s.', packages)
target_setting = ''
# For containers running in Moblab, /usr/local/lib/python2.7/dist-packages/
# is a readonly mount from the host. Therefore, new python modules have to
# be installed in /usr/lib/python2.7/dist-packages/
# Containers created in Moblab does not have autotest/site-packages folder.
if not os.path.exists('/usr/local/autotest/site-packages'):
target_setting = '--target="/usr/lib/python2.7/dist-packages/"'
if python_packages:
utils.run('sudo pip install %s %s' % (target_setting,
' '.join(python_packages)))
logging.debug('Python packages are installed: %s.', python_packages)
def create_from_base(self, name, disable_snapshot_clone=False,
force_cleanup=False):
"""Create a container from the base container.
@param name: Name of the container.
@param disable_snapshot_clone: Set to True to force to clone without
using snapshot clone even if the host supports that.
@param force_cleanup: Force to cleanup existing container.
@return: A Container object for the created container.
@raise ContainerError: If the container already exist.
@raise error.CmdError: If lxc-clone call failed for any reason.
"""
if self.exist(name) and not force_cleanup:
raise error.ContainerError('Container %s already exists.' % name)
use_snapshot = (constants.SUPPORT_SNAPSHOT_CLONE and not
disable_snapshot_clone)
try:
return Container.clone(src=self.base_container,
new_name=name,
new_path=self.container_path,
snapshot=use_snapshot,
cleanup=force_cleanup)
except error.CmdError:
logging.debug('Creating snapshot clone failed. Attempting without '
'snapshot...')
if not use_snapshot:
raise
else:
# Snapshot clone failed, retry clone without snapshot.
container = Container.clone(src=self.base_container,
new_name=name,
new_path=self.container_path,
snapshot=False,
cleanup=force_cleanup)
return container
def _set_lxc_config(self, key, value):
"""Sets an LXC config value for this container.
Configuration changes made while a container is running don't take
effect until the container is restarted. Since this isn't a scenario
that should ever come up in our use cases, calling this method on a
running container will cause a ContainerError.
@param key: The LXC config key to set.
@param value: The value to use for the given key.
@raise error.ContainerError: If the container is already started.
"""
if self.is_running():
raise error.ContainerError(
'_set_lxc_config(%s, %s) called on a running container.' %
(key, value))
config_file = os.path.join(self.container_path, self.name, 'config')
config = '%s = %s' % (key, value)
utils.run(constants.APPEND_CMD_FMT % {
'content': config,
'file': config_file
})
def get_host_ip():
"""Get the IP address of the host running containers on lxcbr*.
This function gets the IP address on network interface lxcbr*. The
assumption is that lxc uses the network interface started with "lxcbr".
@return: IP address of the host running containers.
"""
# The kernel publishes symlinks to various network devices in /sys.
result = utils.run('ls /sys/class/net', ignore_status=True)
# filter out empty strings
interface_names = [x for x in result.stdout.split() if x]
lxc_network = None
for name in interface_names:
if name.startswith('lxcbr'):
lxc_network = name
break
if not lxc_network:
raise error.ContainerError('Failed to find network interface used by '
'lxc. All existing interfaces are: %s' %
interface_names)
netif = interface.Interface(lxc_network)
return netif.ipv4_address
def __init__(self,
container_path,
name,
attribute_values,
src=None,
snapshot=False,
host_path=None):
"""Initialize an object of LXC container with given attribute values.
@param container_path: Directory that stores the container.
@param name: Name of the container.
@param attribute_values: A dictionary of attribute values for the
container.
@param src: An optional source container. If provided, the source
continer is cloned, and the new container will point to the
clone.
@param snapshot: Whether or not to create a snapshot clone. By default,
this is false. If a snapshot is requested and creating
a snapshot clone fails, a full clone will be attempted.
@param host_path: If set to None (the default), a host path will be
generated based on constants.DEFAULT_SHARED_HOST_PATH.
Otherwise, this can be used to override the host path
of the new container, for testing purposes.
"""
# Check if this is a pre-existing LXC container. Do this before calling
# the super ctor, because that triggers container creation.
exists = lxc.get_container_info(container_path, name=name)
super(Zygote, self).__init__(container_path, name, attribute_values,
src, snapshot)
logging.debug('Creating Zygote (lxcpath:%s name:%s)', container_path,
name)
# host_path is a directory within a shared bind-mount, which enables
# bind-mounts from the host system to be shared with the LXC container.
if host_path is not None:
# Allow the host_path to be injected, for testing.
self.host_path = host_path
else:
if exists:
# Pre-existing Zygotes must have a host path.
self.host_path = self._find_existing_host_dir()
if self.host_path is None:
raise error.ContainerError(
'Container %s has no host path.' %
os.path.join(container_path, name))
else:
# New Zygotes use a predefined template to generate a host path.
self.host_path = os.path.join(
os.path.realpath(constants.DEFAULT_SHARED_HOST_PATH),
self.name)
# host_path_ro is a directory for holding intermediate mount points,
# which are necessary when creating read-only bind mounts. See the
# mount_dir method for more details.
#
# Generate a host_path_ro based on host_path.
ro_dir, ro_name = os.path.split(self.host_path.rstrip(os.path.sep))
self.host_path_ro = os.path.join(ro_dir, '%s.ro' % ro_name)
# Remember mounts so they can be cleaned up in destroy.
self.mounts = []
if exists:
self._find_existing_bind_mounts()
else:
# Creating a new Zygote - initialize the host dirs. Don't use sudo,
# so that the resulting directories can be accessed by autoserv (for
# SSP installation, etc).
if not lxc_utils.path_exists(self.host_path):
os.makedirs(self.host_path)
if not lxc_utils.path_exists(self.host_path_ro):
os.makedirs(self.host_path_ro)
# Create the mount point within the container's rootfs.
# Changes within container's rootfs require sudo.
utils.run('sudo mkdir %s' % os.path.join(
self.rootfs, constants.CONTAINER_HOST_DIR.lstrip(os.path.sep)))
self.mount_dir(self.host_path, constants.CONTAINER_HOST_DIR)
def setup_test(self, container_id, job_id, server_package_url, result_path,
control=None, skip_cleanup=False, job_folder=None,
dut_name=None, isolate_hash=None):
"""Setup test container for the test job to run.
The setup includes:
1. Install autotest_server package from given url.
2. Copy over local shadow_config.ini.
3. Mount local site-packages.
4. Mount test result directory.
TODO(dshi): Setup also needs to include test control file for autoserv
to run in container.
@param container_id: ID to assign to the test container.
@param job_id: Job id for the test job to run in the test container.
@param server_package_url: Url to download autotest_server package.
@param result_path: Directory to be mounted to container to store test
results.
@param control: Path to the control file to run the test job. Default is
set to None.
@param skip_cleanup: Set to True to skip cleanup, used to troubleshoot
container failures.
@param job_folder: Folder name of the job, e.g., 123-debug_user.
@param dut_name: Name of the dut to run test, used as the hostname of
the container. Default is None.
@param isolate_hash: String key to look up the isolate package needed
to run test. Default is None, supersedes
server_package_url if present.
@return: A Container object for the test container.
@raise ContainerError: If container does not exist, or not running.
"""
start_time = time.time()
if not os.path.exists(result_path):
raise error.ContainerError('Result directory does not exist: %s',
result_path)
result_path = os.path.abspath(result_path)
# Save control file to result_path temporarily. The reason is that the
# control file in drone_tmp folder can be deleted during scheduler
# restart. For test not using SSP, the window between test starts and
# control file being picked up by the test is very small (< 2 seconds).
# However, for tests using SSP, it takes around 1 minute before the
# container is setup. If scheduler is restarted during that period, the
# control file will be deleted, and the test will fail.
if control:
control_file_name = os.path.basename(control)
safe_control = os.path.join(result_path, control_file_name)
utils.run('cp %s %s' % (control, safe_control))
# Create test container from the base container.
container = self._factory.create_container(container_id)
# Deploy server side package
if isolate_hash:
container.install_ssp_isolate(isolate_hash)
else:
container.install_ssp(server_package_url)
deploy_config_manager = lxc_config.DeployConfigManager(container)
deploy_config_manager.deploy_pre_start()
# Copy over control file to run the test job.
if control:
container.install_control_file(safe_control)
mount_entries = [(constants.SITE_PACKAGES_PATH,
constants.CONTAINER_SITE_PACKAGES_PATH,
True),
(result_path,
os.path.join(constants.RESULT_DIR_FMT % job_folder),
False),
]
# Update container config to mount directories.
for source, destination, readonly in mount_entries:
container.mount_dir(source, destination, readonly)
# Update file permissions.
# TODO(dshi): crbug.com/459344 Skip following action when test container
# can be unprivileged container.
autotest_path = os.path.join(
container.rootfs,
constants.CONTAINER_AUTOTEST_DIR.lstrip(os.path.sep))
utils.run('sudo chown -R root "%s"' % autotest_path)
utils.run('sudo chgrp -R root "%s"' % autotest_path)
container.start(wait_for_network=True)
deploy_config_manager.deploy_post_start()
# Update the hostname of the test container to be `dut-name`.
# Some TradeFed tests use hostname in test results, which is used to
# group test results in dashboard. The default container name is set to
# be the name of the folder, which is unique (as it is composed of job
# id and timestamp. For better result view, the container's hostname is
# set to be a string containing the dut hostname.
if dut_name:
container.set_hostname(constants.CONTAINER_UTSNAME_FORMAT %
dut_name.replace('.', '-'))
container.modify_import_order()
container.verify_autotest_setup(job_folder)
logging.debug('Test container %s is set up.', container.name)
return container
def setup_test(self, name, job_id, server_package_url, result_path,
control=None, skip_cleanup=False):
"""Setup test container for the test job to run.
The setup includes:
1. Install autotest_server package from given url.
2. Copy over local shadow_config.ini.
3. Mount local site-packages.
4. Mount test result directory.
TODO(dshi): Setup also needs to include test control file for autoserv
to run in container.
@param name: Name of the container.
@param job_id: Job id for the test job to run in the test container.
@param server_package_url: Url to download autotest_server package.
@param result_path: Directory to be mounted to container to store test
results.
@param control: Path to the control file to run the test job. Default is
set to None.
@param skip_cleanup: Set to True to skip cleanup, used to troubleshoot
container failures.
@return: A Container object for the test container.
@raise ContainerError: If container does not exist, or not running.
"""
start_time = time.time()
if not os.path.exists(result_path):
raise error.ContainerError('Result directory does not exist: %s',
result_path)
result_path = os.path.abspath(result_path)
# Create test container from the base container.
container = self.create_from_base(name)
# Deploy server side package
usr_local_path = os.path.join(container.rootfs, 'usr', 'local')
autotest_pkg_path = os.path.join(usr_local_path,
'autotest_server_package.tar.bz2')
autotest_path = os.path.join(usr_local_path, 'autotest')
# sudo is required so os.makedirs may not work.
utils.run('sudo mkdir -p %s'% usr_local_path)
download_extract(server_package_url, autotest_pkg_path, usr_local_path)
deploy_config_manager = lxc_config.DeployConfigManager(container)
deploy_config_manager.deploy_pre_start()
# Copy over control file to run the test job.
if control:
container_drone_temp = os.path.join(autotest_path, 'drone_tmp')
utils.run('sudo mkdir -p %s'% container_drone_temp)
container_control_file = os.path.join(
container_drone_temp, os.path.basename(control))
utils.run('sudo cp %s %s' % (control, container_control_file))
if IS_MOBLAB:
site_packages_path = MOBLAB_SITE_PACKAGES
site_packages_container_path = MOBLAB_SITE_PACKAGES_CONTAINER[1:]
else:
site_packages_path = os.path.join(common.autotest_dir,
'site-packages')
site_packages_container_path = os.path.join(
lxc_config.CONTAINER_AUTOTEST_DIR, 'site-packages')
mount_entries = [(site_packages_path, site_packages_container_path,
True),
(os.path.join(common.autotest_dir, 'puppylab'),
os.path.join(lxc_config.CONTAINER_AUTOTEST_DIR,
'puppylab'),
True),
(result_path,
os.path.join(RESULT_DIR_FMT % job_id),
False),
]
# Update container config to mount directories.
for source, destination, readonly in mount_entries:
container.mount_dir(source, destination, readonly)
# Update file permissions.
# TODO(dshi): crbug.com/459344 Skip following action when test container
# can be unprivileged container.
utils.run('sudo chown -R root "%s"' % autotest_path)
utils.run('sudo chgrp -R root "%s"' % autotest_path)
container.start(name)
deploy_config_manager.deploy_post_start()
container.modify_import_order()
container.verify_autotest_setup(job_id)
autotest_es.post(use_http=True,
type_str=CONTAINER_CREATE_METADB_TYPE,
metadata={'drone': socket.gethostname(),
'job_id': job_id,
'time_used': time.time() - start_time,
'success': True})
logging.debug('Test container %s is set up.', name)
return container
def setup_test(self,
name,
job_id,
server_package_url,
result_path,
control=None,
skip_cleanup=False,
job_folder=None,
dut_name=None):
"""Setup test container for the test job to run.
The setup includes:
1. Install autotest_server package from given url.
2. Copy over local shadow_config.ini.
3. Mount local site-packages.
4. Mount test result directory.
TODO(dshi): Setup also needs to include test control file for autoserv
to run in container.
@param name: Name of the container.
@param job_id: Job id for the test job to run in the test container.
@param server_package_url: Url to download autotest_server package.
@param result_path: Directory to be mounted to container to store test
results.
@param control: Path to the control file to run the test job. Default is
set to None.
@param skip_cleanup: Set to True to skip cleanup, used to troubleshoot
container failures.
@param job_folder: Folder name of the job, e.g., 123-debug_user.
@param dut_name: Name of the dut to run test, used as the hostname of
the container. Default is None.
@return: A Container object for the test container.
@raise ContainerError: If container does not exist, or not running.
"""
start_time = time.time()
if not os.path.exists(result_path):
raise error.ContainerError('Result directory does not exist: %s',
result_path)
result_path = os.path.abspath(result_path)
# Save control file to result_path temporarily. The reason is that the
# control file in drone_tmp folder can be deleted during scheduler
# restart. For test not using SSP, the window between test starts and
# control file being picked up by the test is very small (< 2 seconds).
# However, for tests using SSP, it takes around 1 minute before the
# container is setup. If scheduler is restarted during that period, the
# control file will be deleted, and the test will fail.
if control:
control_file_name = os.path.basename(control)
safe_control = os.path.join(result_path, control_file_name)
utils.run('cp %s %s' % (control, safe_control))
# Create test container from the base container.
container = self.create_from_base(name)
# Update the hostname of the test container to be `dut_name`.
# Some TradeFed tests use hostname in test results, which is used to
# group test results in dashboard. The default container name is set to
# be the name of the folder, which is unique (as it is composed of job
# id and timestamp. For better result view, the container's hostname is
# set to be a string containing the dut hostname.
if dut_name:
config_file = os.path.join(container.container_path, name,
'config')
lxc_utsname_setting = (
'lxc.utsname = ' +
CONTAINER_UTSNAME_FORMAT % dut_name.replace('.', '_'))
utils.run(APPEND_CMD_FMT % {
'content': lxc_utsname_setting,
'file': config_file
})
# Deploy server side package
usr_local_path = os.path.join(container.rootfs, 'usr', 'local')
autotest_pkg_path = os.path.join(usr_local_path,
'autotest_server_package.tar.bz2')
autotest_path = os.path.join(usr_local_path, 'autotest')
# sudo is required so os.makedirs may not work.
utils.run('sudo mkdir -p %s' % usr_local_path)
download_extract(server_package_url, autotest_pkg_path, usr_local_path)
deploy_config_manager = lxc_config.DeployConfigManager(container)
deploy_config_manager.deploy_pre_start()
# Copy over control file to run the test job.
if control:
container_drone_temp = os.path.join(autotest_path, 'drone_tmp')
utils.run('sudo mkdir -p %s' % container_drone_temp)
container_control_file = os.path.join(container_drone_temp,
control_file_name)
# Move the control file stored in the result folder to container.
utils.run('sudo mv %s %s' % (safe_control, container_control_file))
if IS_MOBLAB:
site_packages_path = MOBLAB_SITE_PACKAGES
site_packages_container_path = MOBLAB_SITE_PACKAGES_CONTAINER[1:]
else:
site_packages_path = os.path.join(common.autotest_dir,
'site-packages')
site_packages_container_path = os.path.join(
lxc_config.CONTAINER_AUTOTEST_DIR, 'site-packages')
mount_entries = [
(site_packages_path, site_packages_container_path, True),
(os.path.join(common.autotest_dir, 'puppylab'),
os.path.join(lxc_config.CONTAINER_AUTOTEST_DIR,
'puppylab'), True),
(result_path, os.path.join(RESULT_DIR_FMT % job_folder), False),
]
for mount_config in deploy_config_manager.mount_configs:
mount_entries.append((mount_config.source, mount_config.target,
mount_config.readonly))
# Update container config to mount directories.
for source, destination, readonly in mount_entries:
container.mount_dir(source, destination, readonly)
# Update file permissions.
# TODO(dshi): crbug.com/459344 Skip following action when test container
# can be unprivileged container.
utils.run('sudo chown -R root "%s"' % autotest_path)
utils.run('sudo chgrp -R root "%s"' % autotest_path)
container.start(name)
deploy_config_manager.deploy_post_start()
container.modify_import_order()
container.verify_autotest_setup(job_folder)
autotest_es.post(use_http=True,
type_str=CONTAINER_CREATE_METADB_TYPE,
metadata={
'drone': socket.gethostname(),
'job_id': job_id,
'time_used': time.time() - start_time,
'success': True
})
logging.debug('Test container %s is set up.', name)
return container
