def _setup_container_user_noroot(self, user):
""" Setup user for engines without root support.
Equivalent to _setup_container_user() for engines without root support.
"""
host_auth = NixAuthentication()
(passwd, group) = self._select_auth_files()
container_auth = NixAuthentication(passwd, group)
if not user:
user = HostInfo().username()
(valid_user, user_id) = self._user_from_str(user,
host_auth, container_auth)
if not valid_user:
Msg().err("Error: invalid syntax for user", user)
return False
if self.opt["user"] == "root":
self.opt["user"] = HostInfo().username()
self.opt["uid"] = str(HostInfo.uid)
self.opt["gid"] = str(HostInfo.gid)
if (self._is_mountpoint("/etc/passwd") or
self._is_mountpoint("/etc/group")):
self.opt["hostauth"] = self.opt["containerauth"] = False
return True
if self.opt["user"]:
self.opt["uid"] = str(HostInfo.uid)
self.opt["gid"] = str(HostInfo.gid)
else:
if self.opt["hostauth"] or self.opt["containerauth"]:
Msg().err("Error: user not found on host")
return False
self.opt["user"] = user_id["user"] if "user" in user_id else user
self.opt["uid"] = user_id["uid"] if "uid" in user_id else ""
self.opt["gid"] = user_id["gid"] if "gid" in user_id else ""
self._create_user(container_auth, host_auth)
return True
def test_06_cmd_has_option(self):
"""Test06 HostInfo().cmd_has_option."""
status = HostInfo().cmd_has_option("ls", "-a")
self.assertTrue(status)
status = HostInfo().cmd_has_option("ls", "-z")
self.assertFalse(status)
def test_05_oskernel_isgreater(self, mock_kernel):
"""Test05 HostInfo().oskernel_isgreater."""
mock_kernel.return_value = "1.1.2-"
status = HostInfo().oskernel_isgreater([1, 1, 1])
self.assertTrue(status)
mock_kernel.return_value = "1.2.1-"
status = HostInfo().oskernel_isgreater([1, 1, 1])
self.assertTrue(status)
mock_kernel.return_value = "1.0.0-"
status = HostInfo().oskernel_isgreater([1, 1, 1])
self.assertFalse(status)
def _is_same_osenv(self, filename):
"""Check if the host has changed"""
try:
saved = json.loads(FileUtil(filename).getdata())
if (saved["osversion"] == HostInfo().osversion() and
saved["oskernel"] == HostInfo().oskernel() and
saved["arch"] == HostInfo().arch() and
saved["osdistribution"] == \
str(HostInfo().osdistribution())):
return saved
except (IOError, OSError, AttributeError, ValueError, TypeError,
IndexError, KeyError):
pass
return dict()
def _save_osenv(self, filename, save=None):
"""Save host info for is_same_host()"""
if save is None:
save = dict()
try:
save["osversion"] = HostInfo().osversion()
save["oskernel"] = HostInfo().oskernel()
save["arch"] = HostInfo().arch()
save["osdistribution"] = str(HostInfo().osdistribution())
if FileUtil(filename).putdata(json.dumps(save)):
return True
except (AttributeError, ValueError, TypeError,
IndexError, KeyError):
pass
return False
def _is_seccomp_patched(self, executable):
"""Check if kernel has ptrace/seccomp fixes added
on 4.8.0.
Only required for kernels below 4.8.0 to check
if the patch has been backported e.g CentOS 7
"""
if "PROOT_NEW_SECCOMP" in os.environ:
return True
if ("PROOT_NO_SECCOMP" in os.environ or self.proot_noseccomp
or HostInfo().oskernel_isgreater([4, 8, 0])):
return False
host_file = self.container_dir + "/osenv.json"
host_info = self._is_same_osenv(host_file)
if host_info:
if "PROOT_NEW_SECCOMP" in host_info:
return True
return False
out = Uprocess().get_output(
[executable, "-r", "/", executable, "--help"])
if not out:
os.environ["PROOT_NEW_SECCOMP"] = "1"
out = Uprocess().get_output(
[executable, "-r", "/", executable, "--help"])
del os.environ["PROOT_NEW_SECCOMP"]
if out:
self._save_osenv(host_file, dict([
("PROOT_NEW_SECCOMP", 1),
]))
return True
self._save_osenv(host_file)
return False
def select_singularity(self):
"""Set singularity executable and related variables"""
self.executable = Config.conf['use_singularity_executable']
if self.executable != "UDOCKER" and not self.executable:
self.executable = FileUtil("singularity").find_exec()
if self.executable == "UDOCKER" or not self.executable:
self.executable = ""
arch = HostInfo().arch()
image_list = []
if arch == "amd64":
image_list = ["singularity-x86_64", "singularity"]
elif arch == "i386":
image_list = ["singularity-x86", "singularity"]
elif arch == "arm64":
image_list = ["singularity-arm64", "singularity"]
elif arch == "arm":
image_list = ["singularity-arm", "singularity"]
f_util = FileUtil(self.localrepo.bindir)
self.executable = f_util.find_file_in_dir(image_list)
if not os.path.exists(self.executable):
Msg().err("Error: singularity executable not found")
sys.exit(1)
def _run_env_set(self):
"""Environment variables to set"""
self.opt["env"].appendif("HOME=" + self.opt["home"])
self.opt["env"].append("USER="******"user"])
self.opt["env"].append("LOGNAME=" + self.opt["user"])
self.opt["env"].append("USERNAME="******"user"])
if str(self.opt["uid"]) == "0":
self.opt["env"].append(r"PS1=%s# " % self.container_id[:8])
else:
self.opt["env"].append(r"PS1=%s\$ " % self.container_id[:8])
self.opt["env"].append("SHLVL=0")
self.opt["env"].append("container_ruser="******"env"].append("container_root=" + self.container_root)
self.opt["env"].append("container_uuid=" + self.container_id)
self.opt["env"].append("container_execmode=" +
self.exec_mode.get_mode())
cont_name = self.container_names
# if Python 3
if sys.version_info[0] >= 3:
names = str(cont_name).translate(str.maketrans('', '', " '\"[]"))
else:
names = str(cont_name).translate(None, " '\"[]")
self.opt["env"].append("container_names=" + names)
def _apply_whiteouts(self, tarf, destdir):
"""The layered filesystem of docker uses whiteout files
to identify files or directories to be removed.
The format is .wh.<filename>
"""
verbose = ""
if Msg.level >= Msg.VER:
verbose = 'v'
Msg().out("Info: applying whiteouts:", tarf, l=Msg.INF)
wildcards = [
"--wildcards",
]
if not HostInfo().cmd_has_option("tar", wildcards[0]):
wildcards = []
cmd = ["tar", "t" + verbose] + wildcards + ["-f", tarf, r"*/.wh.*"]
whiteouts = Uprocess().get_output(cmd, True)
if not whiteouts:
return
for wh_filename in whiteouts.split('\n'):
if wh_filename:
wh_basename = os.path.basename(wh_filename.strip())
wh_dirname = os.path.dirname(wh_filename)
if wh_basename == ".wh..wh..opq":
if not os.path.isdir(destdir + '/' + wh_dirname):
continue
for f_name in os.listdir(destdir + '/' + wh_dirname):
rm_filename = destdir + '/' \
+ wh_dirname + '/' + f_name
FileUtil(rm_filename).remove(recursive=True)
elif wh_basename.startswith(".wh."):
rm_filename = destdir + '/' \
+ wh_dirname + '/' \
+ wh_basename.replace(".wh.", "", 1)
FileUtil(rm_filename).remove(recursive=True)
return
def select_proot(self):
"""Set proot executable and related variables"""
self.executable = Config.conf['use_proot_executable']
if self.executable != "UDOCKER" and not self.executable:
self.executable = FileUtil("proot").find_exec()
if self.executable == "UDOCKER" or not self.executable:
self.executable = ""
arch = HostInfo().arch()
image_list = []
if arch == "amd64":
if HostInfo().oskernel_isgreater([4, 8, 0]):
image_list = [
"proot-x86_64-4_8_0", "proot-x86_64", "proot"
]
else:
image_list = ["proot-x86_64", "proot"]
elif arch == "i386":
if HostInfo().oskernel_isgreater([4, 8, 0]):
image_list = ["proot-x86-4_8_0", "proot-x86", "proot"]
else:
image_list = ["proot-x86", "proot"]
elif arch == "arm64":
if HostInfo().oskernel_isgreater([4, 8, 0]):
image_list = ["proot-arm64-4_8_0", "proot-arm64", "proot"]
else:
image_list = ["proot-arm64", "proot"]
elif arch == "arm":
if HostInfo().oskernel_isgreater([4, 8, 0]):
image_list = ["proot-arm-4_8_0", "proot-arm", "proot"]
else:
image_list = ["proot-arm", "proot"]
f_util = FileUtil(self.localrepo.bindir)
self.executable = f_util.find_file_in_dir(image_list)
if not os.path.exists(self.executable):
Msg().err("Error: proot executable not found")
sys.exit(1)
if Config.conf['proot_noseccomp'] is not None:
self.proot_noseccomp = Config.conf['proot_noseccomp']
if self.exec_mode.get_mode() == "P2":
self.proot_noseccomp = True
if self._is_seccomp_patched(self.executable):
self.proot_newseccomp = True
def test_01_username(self, mock_getpwuid, mock_uid, mock_gid):
"""Test01 HostInfo().username."""
usr = pwd.struct_passwd(
["root", "*", "0", "0", "root usr", "/root", "/bin/bash"])
mock_uid.return_value = 0
mock_gid.return_value = 0
mock_getpwuid.return_value = usr
name = HostInfo().username()
self.assertEqual(name, usr.pw_name)
def test_02_arch(self, mock_mach, mock_arch):
"""Test02 HostInfo().arch."""
mock_mach.return_value = "x86_64"
mock_arch.return_value = ('64bit', '')
result = HostInfo().arch()
self.assertEqual(result, "amd64")
mock_mach.return_value = "x86_64"
mock_arch.return_value = ('32bit', '')
result = HostInfo().arch()
self.assertEqual(result, "i386")
mock_mach.return_value = "arm"
mock_arch.return_value = ('64bit', '')
result = HostInfo().arch()
self.assertEqual(result, "arm64")
mock_mach.return_value = "arm"
mock_arch.return_value = ('32bit', '')
result = HostInfo().arch()
self.assertEqual(result, "arm")
def _untar_layers(self, tarfiles, destdir):
"""Untar all container layers. Each layer is extracted
and permissions are changed to avoid file permission
issues when extracting the next layer.
"""
if not (tarfiles and destdir):
return False
status = True
gid = str(HostInfo.gid)
optional_flags = [
"--wildcards",
"--delay-directory-restore",
]
for option in optional_flags:
if not HostInfo().cmd_has_option("tar", option):
optional_flags.remove(option)
for tarf in tarfiles:
if tarf != '-':
self._apply_whiteouts(tarf, destdir)
verbose = ''
if Msg.level >= Msg.VER:
verbose = 'v'
Msg().out("Info: extracting:", tarf, l=Msg.INF)
cmd = [
"tar",
"-C",
destdir,
"-x" + verbose,
"--one-file-system",
"--no-same-owner",
"--overwrite",
"--exclude=dev/*",
"--exclude=etc/udev/devices/*",
"--no-same-permissions",
r"--exclude=.wh.*",
] + optional_flags + ["-f", tarf]
if subprocess.call(cmd, stderr=Msg.chlderr, close_fds=True):
Msg().err("Error: while extracting image layer")
status = False
cmd = [
"find", destdir, "(", "-type", "d", "!", "-perm", "-u=x",
"-exec", "chmod", "u+x", "{}", ";", ")", ",", "(", "!",
"-perm", "-u=w", "-exec", "chmod", "u+w", "{}", ";", ")", ",",
"(", "!", "-perm", "-u=r", "-exec", "chmod", "u+r", "{}", ";",
")", ",", "(", "!", "-gid", gid, "-exec", "chgrp", gid, "{}",
";", ")", ",", "(", "-name", ".wh.*", "-exec", "rm", "-f",
"--preserve-root", "{}", ";", ")"
]
if subprocess.call(cmd, stderr=Msg.chlderr, close_fds=True):
status = False
Msg().err("Error: while modifying attributes of image layer")
return status
def namespace_exec(self, method, flags=CLONE_NEWUSER):
"""Execute command in namespace"""
(pread1, pwrite1) = os.pipe()
(pread2, pwrite2) = os.pipe()
cpid = os.fork()
if cpid:
os.close(pwrite1)
os.read(pread1, 1) # wait
user = HostInfo().username()
newidmap = ["newuidmap", str(cpid), "0", str(HostInfo.uid), "1"]
for (subid, subcount) in NixAuthentication().user_in_subuid(user):
newidmap.extend(["1", subid, subcount])
subprocess.call(newidmap)
newidmap = ["newgidmap", str(cpid), "0", str(HostInfo.uid), "1"]
for (subid, subcount) in NixAuthentication().user_in_subgid(user):
newidmap.extend(["1", subid, subcount])
subprocess.call(newidmap)
os.close(pwrite2) # notify
(dummy, status) = os.waitpid(cpid, 0)
if status % 256:
Msg().err("Error: namespace exec action failed")
return False
return True
self.unshare(flags)
os.close(pwrite2)
os.close(pwrite1) # notify
os.read(pread2, 1) # wait
try:
os.setgid(0)
os.setuid(0)
os.setgroups([0, 0, ])
except OSError:
Msg().err("Error: setting ids and groups")
return False
# pylint: disable=protected-access
os._exit(int(method()))
return True
def select_patchelf(self):
"""Set patchelf executable"""
arch = HostInfo().arch()
image_list = list()
if arch == "amd64":
image_list = ["patchelf-x86_64", "patchelf"]
elif arch == "i386":
image_list = ["patchelf-x86", "patchelf"]
elif arch == "arm64":
image_list = ["patchelf-arm64", "patchelf"]
elif arch == "arm":
image_list = ["patchelf-arm", "patchelf"]
f_util = FileUtil(self.localrepo.bindir)
patchelf_exec = f_util.find_file_in_dir(image_list)
if not os.path.exists(patchelf_exec):
Msg().err("Error: patchelf executable not found")
sys.exit(1)
return patchelf_exec
def _run_as_root(self):
"""Set configure running as normal user or as root via --fakeroot
"""
username = HostInfo().username()
if "user" in self.opt:
if self.opt["user"] == username:
return False
if self.opt["user"] != "root" and self.opt["uid"] != '0':
Msg().out("Warning: running as another user not supported")
return False
if self._has_option("--fakeroot", "exec"):
if (NixAuthentication().user_in_subuid(username) and
NixAuthentication().user_in_subgid(username)):
Config.conf['singularity_options'].extend(["--fakeroot", ])
return True
self.opt["user"] = username
return False
def select_runc(self):
"""Set runc executable and related variables"""
self.executable = Config.conf['use_runc_executable']
if self.executable != "UDOCKER" and not self.executable:
self.executable = FileUtil("runc").find_exec()
if self.executable != "UDOCKER" and not self.executable:
self.executable = FileUtil("crun").find_exec()
if self.executable == "UDOCKER" or not self.executable:
self.executable = ""
arch = HostInfo().arch()
image_list = []
eng = ["runc", "crun"]
if "cgroup2" in FileUtil("/proc/filesystems").getdata('r'):
eng = ["crun", "runc"]
if arch == "amd64":
image_list = [
eng[0] + "-x86_64", eng[0], eng[1] + "-x86_64", eng[1]
]
elif arch == "i386":
image_list = [eng[0] + "-x86", eng[0], eng[1] + "-x86", eng[1]]
elif arch == "arm64":
image_list = [
eng[0] + "-arm64", eng[0], eng[1] + "-arm64", eng[1]
]
elif arch == "arm":
image_list = [eng[0] + "-arm", eng[0], eng[1] + "-arm", eng[1]]
f_util = FileUtil(self.localrepo.bindir)
self.executable = f_util.find_file_in_dir(image_list)
if not os.path.exists(self.executable):
Msg().err("Error: runc/crun executable not found")
sys.exit(1)
if "crun" in os.path.basename(self.executable):
self.engine_type = "crun"
elif "runc" in os.path.basename(self.executable):
self.engine_type = "runc"
def run(self, container_id):
"""Execute a Docker container using runc. This is the main method
invoked to run the a container with runc.
* argument: container_id or name
* options: many via self.opt see the help
"""
Config.conf['sysdirs_list'] = (
"/etc/resolv.conf",
"/etc/host.conf",
# "/etc/passwd", "/etc/group",
)
# setup execution
if not self._run_init(container_id):
return 2
self._run_invalid_options()
self._container_specfile = "config.json"
if self.container_dir:
if self.localrepo.iswriteable_container(container_id):
self._container_specdir = self.container_dir
else:
self._container_specdir = FileUtil("SPECDIR").mktmpdir()
FileUtil(self._container_specdir).register_prefix()
self._container_specfile = \
self._container_specdir + '/' + self._container_specfile
self._filebind = FileBind(self.localrepo, container_id)
self._filebind.setup()
self.select_runc()
# create new OCI spec file
if not self._load_spec(new=True):
return 4
self._uid_check()
# if not --hostenv clean the environment
self._run_env_cleanup_list()
# set environment variables
self._run_env_set()
self._set_spec()
if (Config.conf['runc_nomqueue']
or (Config.conf['runc_nomqueue'] is None
and not HostInfo().oskernel_isgreater([4, 8, 0]))):
self._del_mount_spec("mqueue", "/dev/mqueue")
self._del_mount_spec("cgroup", "/sys/fs/cgroup")
self._del_namespace_spec("network")
self._set_id_mappings()
self._add_volume_bindings()
self._add_devices()
self._add_capabilities_spec()
self._mod_mount_spec("shm", "/dev/shm", {"options": ["size=2g"]})
self._proot_overlay()
self._save_spec()
if Msg.level >= Msg.DBG:
runc_debug = [
"--debug",
]
Msg().out(
json.dumps(self._container_specjson, indent=4, sort_keys=True))
else:
runc_debug = []
# build the actual command
self.execution_id = Unique().uuid(self.container_id)
cmd_l = self._set_cpu_affinity()
cmd_l.append(self.executable)
cmd_l.extend(runc_debug)
cmd_l.extend(["--root", self._container_specdir, "run"])
cmd_l.extend(["--bundle", self._container_specdir, self.execution_id])
Msg().err("CMD =", cmd_l, l=Msg.VER)
self._run_banner(self.opt["cmd"][0], '%')
if sys.stdout.isatty():
return self.run_pty(cmd_l)
return self.run_nopty(cmd_l)
def _has_option(self, search_option, arg=None):
"""Check if executable has a given cli option"""
return HostInfo().cmd_has_option(self.executable, search_option, arg)
def run(self, container_id):
"""Execute a Docker container using PRoot. This is the main method
invoked to run the a container with PRoot.
* argument: container_id or name
* options: many via self.opt see the help
"""
# setup execution
if not self._run_init(container_id):
return 2
self.select_proot()
# seccomp and ptrace behavior change on 4.8.0 onwards
if self.proot_noseccomp or os.getenv("PROOT_NO_SECCOMP"):
self.opt["env"].append("PROOT_NO_SECCOMP=1")
if self.proot_newseccomp or os.getenv("PROOT_NEW_SECCOMP"):
self.opt["env"].append("PROOT_NEW_SECCOMP=1")
if not HostInfo().oskernel_isgreater([3, 0, 0]):
self._kernel = "6.0.0"
if self.opt["kernel"]:
self._kernel = self.opt["kernel"]
# set environment variables
self._run_env_set()
if Msg.level >= Msg.DBG:
proot_verbose = [
"-v",
"9",
]
else:
proot_verbose = []
if (Config.conf['proot_killonexit']
and self._has_option("--kill-on-exit")):
proot_kill_on_exit = [
"--kill-on-exit",
]
else:
proot_kill_on_exit = []
# build the actual command
cmd_l = self._set_cpu_affinity()
cmd_l.append(self.executable)
cmd_l.extend(proot_verbose)
cmd_l.extend(proot_kill_on_exit)
cmd_l.extend(self._get_volume_bindings())
cmd_l.extend(self._set_uid_map())
cmd_l.extend([
"-k",
self._kernel,
])
cmd_l.extend(self._get_network_map())
cmd_l.extend([
"-r",
self.container_root,
])
if self.opt["cwd"]: # set current working directory
cmd_l.extend([
"-w",
self.opt["cwd"],
])
cmd_l.extend(self.opt["cmd"])
Msg().out("CMD =", cmd_l, l=Msg.VER)
# cleanup the environment
self._run_env_cleanup_dict()
# execute
self._run_banner(self.opt["cmd"][0])
status = subprocess.call(cmd_l,
shell=False,
close_fds=True,
env=os.environ.update(self.opt["env"].dict()))
return status
def test_07_termsize(self, mock_chkout):
"""Test07 HostInfo().termsize."""
mock_chkout.return_value = "24 80"
status = HostInfo().termsize()
self.assertEqual(status, (24, 80))
def __init__(self, localrepo, exec_mode):
super(PRootEngine, self).__init__(localrepo, exec_mode)
self.executable = None # PRoot
self.proot_noseccomp = False # No seccomp mode
self.proot_newseccomp = False # New seccomp mode
self._kernel = HostInfo().oskernel() # Emulate kernel
def create_container_meta(self, layer_id, comment="created by udocker"):
"""Create metadata for a given container layer, used in import.
A file for import is a tarball of a directory tree, does not contain
metadata. This method creates minimal metadata.
"""
container_json = dict()
container_json["id"] = layer_id
container_json["comment"] = comment
container_json["created"] = \
time.strftime("%Y-%m-%dT%H:%M:%S.000000000Z")
container_json["architecture"] = HostInfo().arch()
container_json["os"] = HostInfo().osversion()
layer_file = self.localrepo.layersdir + '/' + layer_id + ".layer"
container_json["size"] = FileUtil(layer_file).size()
if container_json["size"] == -1:
container_json["size"] = 0
container_json["container_config"] = {
"Hostname": "",
"Domainname": "",
"User": "",
"Memory": 0,
"MemorySwap": 0,
"CpusShares": 0,
"Cpuset": "",
"AttachStdin": False,
"AttachStdout": False,
"AttachStderr": False,
"PortSpecs": None,
"ExposedPorts": None,
"Tty": False,
"OpenStdin": False,
"StdinOnce": False,
"Env": None,
"Cmd": None,
"Image": "",
"Volumes": None,
"WorkingDir": "",
"Entrypoint": None,
"NetworkDisable": False,
"MacAddress": "",
"OnBuild": None,
"Labels": None
}
container_json["config"] = {
"Hostname": "",
"Domainname": "",
"User": "",
"Memory": 0,
"MemorySwap": 0,
"CpusShares": 0,
"Cpuset": "",
"AttachStdin": False,
"AttachStdout": False,
"AttachStderr": False,
"PortSpecs": None,
"ExposedPorts": None,
"Tty": False,
"OpenStdin": False,
"StdinOnce": False,
"Env": None,
"Cmd": None,
"Image": "",
"Volumes": None,
"WorkingDir": "",
"Entrypoint": None,
"NetworkDisable": False,
"MacAddress": "",
"OnBuild": None,
"Labels": None
}
return container_json
def test_04_oskernel(self, mock_rel):
"""Test04 HostInfo().oskernel."""
mock_rel.return_value = "3.2.1"
result = HostInfo().oskernel()
self.assertEqual(result, "3.2.1")
def test_03_osversion(self, mock_sys):
"""Test03 HostInfo().osversion."""
mock_sys.return_value = "linux"
result = HostInfo().osversion()
self.assertEqual(result, "linux")