def mount_bind(newroot, target, source=None, recursive=True, read_only=True):
"""Bind mounts `source` to `newroot/target` so that `source` is accessed
when reaching `newroot/target`.
If a directory, the source will be mounted using --rbind.
"""
# Ensure root directory exists
if not os.path.exists(newroot):
raise exc.ContainerSetupError('Path %r does not exist' % newroot)
if source is None:
source = target
target = fs.norm_safe(target)
source = fs.norm_safe(source)
# Make sure target directory exists.
if not os.path.exists(source):
raise exc.ContainerSetupError('Source path %r does not exist' % source)
mnt_flags = [mount.MS_BIND]
# Use --rbind for directories and --bind for files.
if recursive and os.path.isdir(source):
mnt_flags.append(mount.MS_REC)
# Strip leading /, ensure that mount is relative path.
while target.startswith('/'):
target = target[1:]
# Create mount directory, make sure it does not exists.
target_fp = os.path.join(newroot, target)
if os.path.isdir(source):
fs.mkdir_safe(target_fp)
else:
fs.mkfile_safe(target_fp)
res = mount.mount(source=source,
target=target_fp,
fs_type=None,
mnt_flags=mnt_flags)
if res == 0 and read_only:
res = mount.mount(source=None,
target=target_fp,
fs_type=None,
mnt_flags=(mount.MS_BIND, mount.MS_RDONLY,
mount.MS_REMOUNT))
return res
def unpack(self, container_dir, root_dir, app, app_cgroups, data):
root_dir = fs.norm_safe(root_dir)
emptydirs, stickydirs, mounts = configure()
for plugin in image_fs.plugins(app):
_LOGGER.info('Processing plugin: %r', plugin)
extra = plugin(self.tm_env).configure(container_dir, app)
if extra is not None:
emptydirs.update(extra[0])
stickydirs.update(extra[1])
mounts.update(extra[2])
make_fsroot(root_dir, emptydirs, stickydirs, mounts)
make_osroot(root_dir, app, data)
# FIXME: Lots of things are still reading this file.
# Copy updated state manifest as app.json in the
# container_dir so it is visible in chrooted env.
shutil.copy(os.path.join(container_dir, runtime.STATE_JSON),
os.path.join(root_dir, appcfg.APP_JSON))
cgrp = os.path.join(app_cgroups['memory'], 'services')
create_environ_dir(container_dir, root_dir, app)
create_supervision_tree(
self.tm_env, container_dir, root_dir, app,
cgroups_path=cgroups.makepath(
'freezer', cgrp
),
)
create_overlay(self.tm_env, container_dir, root_dir, app)
def configure(self, container_dir, app):
root_dir = os.path.join(container_dir, 'root')
newroot_norm = fs.norm_safe(root_dir)
mounts = [
]
emptydirs = [
'/u',
'/var/account',
'/var/empty',
'/var/lock',
'/var/log',
'/var/run',
]
stickydirs = [
'/opt',
]
for mount in mounts:
if os.path.exists(mount):
fs.mount_bind(newroot_norm, mount)
for directory in emptydirs:
fs.mkdir_safe(newroot_norm + directory)
for directory in stickydirs:
os.chmod(newroot_norm + directory, 0o777 | stat.S_ISVTX)
def configure(_approot, newroot, _app):
"""Configure layout in chroot."""
newroot_norm = fs.norm_safe(newroot)
mounts = [
]
emptydirs = [
'/u',
'/var/account',
'/var/empty',
'/var/lock',
'/var/log',
'/var/run',
]
stickydirs = [
'/opt',
]
for mount in mounts:
if os.path.exists(mount):
fs.mount_bind(newroot_norm, mount)
for directory in emptydirs:
fs.mkdir_safe(newroot_norm + directory)
for directory in stickydirs:
os.chmod(newroot_norm + directory, 0777 | stat.S_ISVTX)
def configure(self, container_dir, app):
root_dir = os.path.join(container_dir, 'root')
newroot_norm = fs.norm_safe(root_dir)
emptydirs = [
'/opt/s6',
'/opt/treadmill',
'/opt/treadmill-bind',
]
stickydirs = []
mounts = [
'/opt/s6',
'/opt/treadmill',
'/opt/treadmill-bind',
]
for directory in emptydirs:
fs.mkdir_safe(newroot_norm + directory)
for directory in stickydirs:
os.chmod(newroot_norm + directory, 0o777 | stat.S_ISVTX)
for mount in mounts:
if os.path.exists(mount):
fs_linux.mount_bind(newroot_norm,
mount,
recursive=True,
read_only=True)
def make_root(root_dir):
"""create container root by calling pivot_root
"""
newroot_norm = fs.norm_safe(root_dir)
old_pivot = os.path.join(newroot_norm, PIVOT_PATH)
fs.mkdir_safe(old_pivot)
proot.pivot_root(newroot_norm, old_pivot)
# return list of mount to be umounted from down to top
to_umount = move_mounts(os.path.sep + PIVOT_PATH)
# umount entries from down to top
for mount_entry in to_umount:
_umount_with_detach(mount_entry.target)
def _create_root_dir(container_dir, localdisk):
"""Prepares chrooted environment."""
# Create root directory structure (chroot base).
# container_dir/<subdir>
root_dir = os.path.join(container_dir, 'root')
already_initialized = fs_linux.blk_fs_test(localdisk['block_dev'])
if not already_initialized:
# Format the block device
fs_linux.blk_fs_create(localdisk['block_dev'])
_LOGGER.info('Creating container root directory: %s', root_dir)
# Creates directory that will serve as new root.
fs.mkdir_safe(fs.norm_safe(root_dir))
# Unshare the mount namespace
unshare.unshare(unshare.CLONE_NEWNS)
# Mount the container root volume
fs_linux.mount_filesystem(localdisk['block_dev'], root_dir)
return root_dir
def _create_root_dir(tm_env, container_dir, root_dir, app):
"""Prepares chrooted environment."""
# Generate a unique name for the app
unique_name = appcfg.app_unique_name(app)
# First wait for the block device to be ready
localdisk_client = tm_env.svc_localdisk.make_client(
os.path.join(container_dir, 'localdisk'))
localdisk = localdisk_client.wait(unique_name)
already_initialized = fs.test_filesystem(localdisk['block_dev'])
if not already_initialized:
# Format the block device
fs.create_filesystem(localdisk['block_dev'])
_LOGGER.info('Creating container root directory: %s', root_dir)
# Creates directory that will serve as new root.
fs.mkdir_safe(fs.norm_safe(root_dir))
# Unshare the mount namespace
unshare.unshare(unshare.CLONE_NEWNS)
# Mount the container root volume
fs.mount_filesystem(localdisk['block_dev'], root_dir)
def make_fsroot(root, proid):
"""Initializes directory structure for the container in a new root.
- Bind directories in parent / (with exceptions - see below.)
- Skip /tmp, create /tmp in the new root with correct permissions.
- Selectively create / bind /var.
- /var/tmp (new)
- /var/logs (new)
- /var/spool - create empty with dirs.
- Bind everything in /var, skipping /spool/tickets
"""
newroot_norm = fs.norm_safe(root)
mounts = [
'/bin',
'/common',
'/dev',
'/etc',
'/home',
'/lib',
'/lib64',
'/mnt',
'/proc',
'/sbin',
'/srv',
'/sys',
'/usr',
'/var/tmp/treadmill/env',
'/var/tmp/treadmill/spool',
] + glob.glob('/opt/*')
emptydirs = [
'/tmp',
'/opt',
'/var/empty',
'/var/run',
'/var/spool/keytabs',
'/var/spool/tickets',
'/var/spool/tokens',
'/var/tmp',
'/var/tmp/cores',
]
stickydirs = [
'/tmp',
'/opt',
'/var/spool/keytabs',
'/var/spool/tickets',
'/var/spool/tokens',
'/var/tmp',
'/var/tmp/cores/',
]
for directory in emptydirs:
_LOGGER.debug('Creating empty dir: %s', directory)
fs.mkdir_safe(newroot_norm + directory)
for directory in stickydirs:
os.chmod(newroot_norm + directory, 0o777 | stat.S_ISVTX)
for mount in mounts:
if os.path.exists(mount):
fs.mount_bind(newroot_norm, mount)
# Mount .../tickets .../keytabs on tempfs, so that they will be cleaned
# up when the container exits.
#
# TODO: Do we need to have a single mount for all tmpfs dirs?
for tmpfsdir in [
'/var/spool/tickets', '/var/spool/keytabs', '/var/spool/tokens'
]:
fs.mount_tmpfs(newroot_norm, tmpfsdir, '4M')
def make_fsroot(root_dir, app, data):
"""Initializes directory structure for the container in a new root.
The container uses pretty much a blank a FHS 3 layout.
- Bind directories in parent / (with exceptions - see below.)
- Skip /tmp, create /tmp in the new root with correct permissions.
- Selectively create / bind /var.
- /var/tmp (new)
- /var/log (new)
- /var/spool - create empty with dirs.
- Bind everything in /var, skipping /spool/tickets
:param ``str`` root_dit:
Container root directory.
:param app:
Container app manifest.
:param ``dict`` data:
Local configuration data.
"""
newroot_norm = fs.norm_safe(root_dir)
emptydirs = [
'/bin',
'/dev',
'/etc',
'/home',
'/lib',
'/lib64',
'/opt',
'/proc',
'/root',
'/run',
'/sbin',
'/sys',
'/tmp',
'/usr',
'/var/cache',
'/var/empty',
'/var/empty/sshd',
'/var/lib',
'/var/lock',
'/var/log',
'/var/opt',
'/var/spool',
'/var/tmp',
'/var/spool/keytabs',
'/var/spool/tickets',
'/var/spool/tokens',
# for SSS
'/var/lib/sss',
]
stickydirs = [
'/opt',
'/run',
'/tmp',
'/var/cache',
'/var/lib',
'/var/lock',
'/var/log',
'/var/opt',
'/var/tmp',
'/var/spool/keytabs',
'/var/spool/tickets',
'/var/spool/tokens',
]
# these folders are shared with underlying host and other containers,
mounts = [
'/bin',
'/etc', # TODO: Add /etc/opt
'/lib',
'/lib64',
'/root',
'/sbin',
'/usr',
# for SSS
'/var/lib/sss',
# TODO: Remove below once PAM UDS is implemented
os.path.expandvars('${TREADMILL_APPROOT}/env'),
os.path.expandvars('${TREADMILL_APPROOT}/spool'),
]
for directory in emptydirs:
fs.mkdir_safe(newroot_norm + directory)
for directory in stickydirs:
os.chmod(newroot_norm + directory, 0o777 | stat.S_ISVTX)
# /var/empty must be owned by root and not group or world-writable.
os.chmod(os.path.join(newroot_norm, 'var/empty'), 0o711)
# Mount a new sysfs for the container, bring in the /sys/fs subtree from
# the host.
fs_linux.mount_sysfs(newroot_norm)
fs_linux.mount_bind(newroot_norm,
os.path.join(os.sep, 'sys', 'fs'),
recursive=True,
read_only=False)
make_dev(newroot_norm)
# Passthrough node devices per the node config data
extra_devices = data.get('runtime', {}).get('passthrough_devices', [])
make_extra_dev(newroot_norm, extra_devices, app.proid)
# Per FHS3 /var/run should be a symlink to /run which should be tmpfs
fs.symlink_safe(os.path.join(newroot_norm, 'var', 'run'), '/run')
# We create an unbounded tmpfs mount so that runtime data can be written to
# it, counting against the memory limit of the container.
fs_linux.mount_tmpfs(newroot_norm, '/run')
# Make shared directories/files readonly to container
for mount in mounts:
if os.path.exists(mount):
fs_linux.mount_bind(newroot_norm,
mount,
recursive=True,
read_only=True)
# /etc/docker is a file neceesary for docker daemon
_docker.mount_docker_daemon_path(newroot_norm, app)
def make_fsroot(root_dir, app):
"""Initializes directory structure for the container in a new root.
The container uses pretty much a blank a FHS 3 layout.
- Bind directories in parent / (with exceptions - see below.)
- Skip /tmp, create /tmp in the new root with correct permissions.
- Selectively create / bind /var.
- /var/tmp (new)
- /var/log (new)
- /var/spool - create empty with dirs.
- Bind everything in /var, skipping /spool/tickets
tm_env is used to deliver abort events
"""
newroot_norm = fs.norm_safe(root_dir)
emptydirs = [
'/bin',
'/dev',
'/etc',
'/home',
'/lib',
'/lib64',
'/opt',
'/proc',
'/root',
'/run',
'/sbin',
'/sys',
'/tmp',
'/usr',
'/var/cache',
'/var/empty',
'/var/empty/sshd',
'/var/lib',
'/var/lock',
'/var/log',
'/var/opt',
'/var/spool',
'/var/tmp',
'/var/spool/keytabs',
'/var/spool/tickets',
'/var/spool/tokens',
# for SSS
'/var/lib/sss',
]
stickydirs = [
'/opt',
'/run',
'/tmp',
'/var/cache',
'/var/lib',
'/var/lock',
'/var/log',
'/var/opt',
'/var/tmp',
'/var/spool/keytabs',
'/var/spool/tickets',
'/var/spool/tokens',
]
# these folders are shared with underlying host and other containers,
mounts = [
'/bin',
'/etc', # TODO: Add /etc/opt
'/lib',
'/lib64',
'/root',
'/sbin',
'/usr',
# for SSS
'/var/lib/sss',
# TODO: Remove below once PAM UDS is implemented
os.path.expandvars('${TREADMILL_APPROOT}/env'),
os.path.expandvars('${TREADMILL_APPROOT}/spool'),
]
for directory in emptydirs:
fs.mkdir_safe(newroot_norm + directory)
for directory in stickydirs:
os.chmod(newroot_norm + directory, 0o777 | stat.S_ISVTX)
# /var/empty must be owned by root and not group or world-writable.
os.chmod(os.path.join(newroot_norm, 'var/empty'), 0o711)
fs_linux.mount_bind(newroot_norm,
os.path.join(os.sep, 'sys'),
source='/sys',
recursive=True,
read_only=False)
make_dev(newroot_norm)
# Per FHS3 /var/run should be a symlink to /run which should be tmpfs
fs.symlink_safe(os.path.join(newroot_norm, 'var', 'run'), '/run')
# We create an unbounded tmpfs mount so that runtime data can be written to
# it, counting against the memory limit of the container.
fs_linux.mount_tmpfs(newroot_norm, '/run')
# Make shared directories/files readonly to container
for mount in mounts:
if os.path.exists(mount):
fs_linux.mount_bind(newroot_norm,
mount,
recursive=True,
read_only=True)
if hasattr(app, 'docker') and app.docker:
# If unable to mount docker directory, we throw Aborted events
try:
_mount_docker_tmpfs(newroot_norm)
except FileNotFoundError as err:
_LOGGER.error('Failed to mount docker tmpfs: %s', err)
# this exception is caught by sproc run to generate abort event
raise exc.ContainerSetupError(
msg=str(err),
reason=app_abort.AbortedReason.UNSUPPORTED,
)
def make_fsroot(root_dir):
"""Initializes directory structure for the container in a new root.
The container uses pretty much a blank a FHS 3 layout.
- Bind directories in parent / (with exceptions - see below.)
- Skip /tmp, create /tmp in the new root with correct permissions.
- Selectively create / bind /var.
- /var/tmp (new)
- /var/log (new)
- /var/spool - create empty with dirs.
- Bind everything in /var, skipping /spool/tickets
"""
newroot_norm = fs.norm_safe(root_dir)
emptydirs = [
'/bin',
'/dev',
'/etc',
'/home',
'/lib',
'/lib64',
'/opt',
'/proc',
'/root',
'/run',
'/sbin',
'/sys',
'/tmp',
'/usr',
'/var/cache',
'/var/empty',
'/var/lib',
'/var/lock',
'/var/log',
'/var/opt',
'/var/spool',
'/var/tmp',
'/var/spool/keytabs',
'/var/spool/tickets',
'/var/spool/tokens',
# for SSS
'/var/lib/sss',
]
stickydirs = [
'/opt',
'/run',
'/tmp',
'/var/cache',
'/var/lib',
'/var/lock',
'/var/log',
'/var/opt',
'/var/tmp',
'/var/spool/keytabs',
'/var/spool/tickets',
'/var/spool/tokens',
]
# these folders are shared with underlying host and other containers,
mounts = [
'/bin',
'/etc', # TODO: Add /etc/opt
'/lib',
'/lib64',
'/root',
'/sbin',
'/usr',
# for SSS
'/var/lib/sss',
# TODO: Remove below once PAM UDS is implemented
'/var/tmp/treadmill/env',
'/var/tmp/treadmill/spool',
]
# Add everything under /opt
mounts += glob.glob('/opt/*')
for directory in emptydirs:
fs.mkdir_safe(newroot_norm + directory)
for directory in stickydirs:
os.chmod(newroot_norm + directory, 0o777 | stat.S_ISVTX)
# /var/empty must be owned by root and not group or world-writable.
os.chmod(os.path.join(newroot_norm, 'var/empty'), 0o711)
fs_linux.mount_bind(newroot_norm,
os.path.join(os.sep, 'sys'),
source='/sys',
recursive=True,
read_only=False)
# TODO: For security, /dev/ should be minimal and separated to each
# container.
fs_linux.mount_bind(newroot_norm,
os.path.join(os.sep, 'dev'),
source='/dev',
recursive=True,
read_only=False)
# Per FHS3 /var/run should be a symlink to /run which should be tmpfs
fs.symlink_safe(os.path.join(newroot_norm, 'var', 'run'), '/run')
# We create an unbounded tmpfs mount so that runtime data can be written to
# it, counting against the memory limit of the container.
fs_linux.mount_tmpfs(newroot_norm, '/run')
# Make shared directories/files readonly to container
for mount in mounts:
if os.path.exists(mount):
fs_linux.mount_bind(newroot_norm,
mount,
recursive=True,
read_only=True)
