def __init__(self, topdir=None):
self.topdir = topdir if topdir else Config.conf['topdir']
self.bindir = Config.conf['bindir']
self.libdir = Config.conf['libdir']
self.docdir = Config.conf['docdir']
self.reposdir = Config.conf['reposdir']
self.layersdir = Config.conf['layersdir']
self.containersdir = Config.conf['containersdir']
self.homedir = Config.conf['homedir']
if not self.bindir:
self.bindir = self.topdir + "/bin"
if not self.libdir:
self.libdir = self.topdir + "/lib"
if not self.docdir:
self.docdir = self.topdir + "/doc"
if not self.reposdir:
self.reposdir = self.topdir + "/repos"
if not self.layersdir:
self.layersdir = self.topdir + "/layers"
if not self.containersdir:
self.containersdir = self.topdir + "/containers"
self.cur_repodir = ""
self.cur_tagdir = ""
self.cur_containerdir = ""
FileUtil(self.reposdir).register_prefix()
FileUtil(self.layersdir).register_prefix()
FileUtil(self.containersdir).register_prefix()
def patch_ld(self, output_elf=None):
"""Patch ld.so"""
elf_loader = self.get_container_loader()
if FileUtil(self._container_ld_so_orig).size() == -1:
status = FileUtil(elf_loader).copyto(self._container_ld_so_orig)
if not status:
return False
ld_data = FileUtil(self._container_ld_so_orig).getdata('rb')
if not ld_data:
ld_data = FileUtil(elf_loader).getdata('rb')
if not ld_data:
return False
nul_etc = "\x00/\x00\x00\x00\x00\x00\x00\x00\x00\x00".encode()
nul_lib = "\x00/\x00\x00\x00".encode()
nul_usr = "******".encode()
etc = "\x00/etc/ld.so".encode()
lib = "\x00/lib".encode()
usr = "******".encode()
ld_data = ld_data.replace(etc, nul_etc).\
replace(lib, nul_lib).replace(usr, nul_usr)
ld_library_path_orig = "\x00LD_LIBRARY_PATH\x00".encode()
ld_library_path_new = "\x00LD_LIBRARY_REAL\x00".encode()
ld_data = ld_data.replace(ld_library_path_orig, ld_library_path_new)
if output_elf is None:
return bool(FileUtil(elf_loader).putdata(ld_data, 'wb'))
return bool(FileUtil(output_elf).putdata(ld_data, 'wb'))
def load(self, imagefile, imagerepo=None):
"""Generic load of image tags to a file"""
if not os.path.exists(imagefile) and imagefile != '-':
Msg().err("Error: image file does not exist:", imagefile)
return False
tmp_imagedir = FileUtil("load").mktmp()
try:
os.makedirs(tmp_imagedir)
except (IOError, OSError):
return False
if not self._untar_saved_container(imagefile, tmp_imagedir):
Msg().err("Error: failed to extract container:", imagefile)
FileUtil(tmp_imagedir).remove(recursive=True)
return False
imagetype = self._get_imagedir_type(tmp_imagedir)
if imagetype == "Docker":
repositories = DockerLocalFileAPI(self.localrepo).load(
tmp_imagedir, imagerepo)
elif imagetype == "OCI":
repositories = OciLocalFileAPI(self.localrepo).load(
tmp_imagedir, imagerepo)
else:
repositories = []
FileUtil(tmp_imagedir).remove(recursive=True)
return repositories
def set_version(self, version):
"""Set the version of the image TAG repository currently
it supports Docker images with versions v1 and v2
to be invoked after setup_tag()
"""
if not (self.cur_repodir and self.cur_tagdir):
return False
if not os.path.exists(self.cur_repodir):
return False
if not os.path.exists(self.cur_tagdir):
return False
directory = self.cur_tagdir
if (os.path.exists(directory + "/v1") and version != "v1"
or os.path.exists(directory + "/v2") and version != "v2"):
if len(os.listdir(directory)) == 1:
try:
FileUtil(directory + "/v1").remove()
FileUtil(directory + "/v2").remove()
except (IOError, OSError):
pass
if os.listdir(directory):
return False
try:
# Create version file
open(directory + "/" + version, 'a').close()
except (IOError, OSError):
return False
return True
def add_user(self, user, passw, uid, gid, gecos, home, shell):
"""Add a *nix user to a /etc/passwd file"""
line = "%s:%s:%s:%s:%s:%s:%s\n" % \
(user, passw, uid, gid, gecos, home, shell)
if line in FileUtil(self.passwd_file).getdata('r'):
return True
return FileUtil(self.passwd_file).putdata(line, 'a')
def select_singularity(self):
"""Set singularity executable and related variables"""
self.executable = Config.conf['use_singularity_executable']
if self.executable != "UDOCKER" and not self.executable:
self.executable = FileUtil("singularity").find_exec()
if self.executable == "UDOCKER" or not self.executable:
self.executable = ""
arch = HostInfo().arch()
image_list = []
if arch == "amd64":
image_list = ["singularity-x86_64", "singularity"]
elif arch == "i386":
image_list = ["singularity-x86", "singularity"]
elif arch == "arm64":
image_list = ["singularity-arm64", "singularity"]
elif arch == "arm":
image_list = ["singularity-arm", "singularity"]
f_util = FileUtil(self.localrepo.bindir)
self.executable = f_util.find_file_in_dir(image_list)
if not os.path.exists(self.executable):
Msg().err("Error: singularity executable not found")
sys.exit(1)
def save(self, imagetag_list, imagefile):
"""Save a set of image tags to a file similarly to docker save
"""
tmp_imagedir = FileUtil("save").mktmp()
try:
os.makedirs(tmp_imagedir)
except (IOError, OSError):
return False
structure = dict()
structure["manifest"] = []
structure["repositories"] = dict()
status = False
for (imagerepo, tag) in imagetag_list:
status = self._save_image(imagerepo, tag, structure, tmp_imagedir)
if not status:
Msg().err("Error: save image failed:", imagerepo + ':' + tag)
break
if status:
self.localrepo.save_json(tmp_imagedir + "/manifest.json",
structure["manifest"])
self.localrepo.save_json(tmp_imagedir + "/repositories",
structure["repositories"])
if not FileUtil(tmp_imagedir).tar(imagefile):
Msg().err("Error: save image failed in writing tar", imagefile)
status = False
else:
Msg().err("Error: no images specified")
FileUtil(tmp_imagedir).remove(recursive=True)
return status
def _load_spec(self, new=False):
"""Generate runc spec file"""
if FileUtil(self._container_specfile).size() != -1 and new:
FileUtil(self._container_specfile).register_prefix()
FileUtil(self._container_specfile).remove()
if FileUtil(self._container_specfile).size() == -1:
cmd_l = [
self.executable,
"spec",
"--rootless",
]
status = subprocess.call(cmd_l, shell=False, stderr=Msg.chlderr,
close_fds=True,
cwd=os.path.realpath(\
self._container_specdir))
if status:
return False
json_obj = None
infile = None
try:
infile = open(self._container_specfile, 'r')
json_obj = json.load(infile)
except (IOError, OSError, AttributeError, ValueError, TypeError):
json_obj = None
if infile:
infile.close()
self._container_specjson = json_obj
return json_obj
def _apply_whiteouts(self, tarf, destdir):
"""The layered filesystem of docker uses whiteout files
to identify files or directories to be removed.
The format is .wh.<filename>
"""
verbose = ""
if Msg.level >= Msg.VER:
verbose = 'v'
Msg().out("Info: applying whiteouts:", tarf, l=Msg.INF)
wildcards = [
"--wildcards",
]
if not HostInfo().cmd_has_option("tar", wildcards[0]):
wildcards = []
cmd = ["tar", "t" + verbose] + wildcards + ["-f", tarf, r"*/.wh.*"]
whiteouts = Uprocess().get_output(cmd, True)
if not whiteouts:
return
for wh_filename in whiteouts.split('\n'):
if wh_filename:
wh_basename = os.path.basename(wh_filename.strip())
wh_dirname = os.path.dirname(wh_filename)
if wh_basename == ".wh..wh..opq":
if not os.path.isdir(destdir + '/' + wh_dirname):
continue
for f_name in os.listdir(destdir + '/' + wh_dirname):
rm_filename = destdir + '/' \
+ wh_dirname + '/' + f_name
FileUtil(rm_filename).remove(recursive=True)
elif wh_basename.startswith(".wh."):
rm_filename = destdir + '/' \
+ wh_dirname + '/' \
+ wh_basename.replace(".wh.", "", 1)
FileUtil(rm_filename).remove(recursive=True)
return
def get_original_loader(self):
"""Get the pathname of the original ld.so"""
if os.path.exists(self._container_ld_so_path):
return FileUtil(self._container_ld_so_path).getdata('r').strip()
elf_loader = self.guess_elf_loader()
if elf_loader:
FileUtil(self._container_ld_so_path).putdata(elf_loader, 'w')
return elf_loader
def test_07_rmdir(self, mock_rmdir):
"""Test07 FileUtil.rmdir()."""
mock_rmdir.return_value = None
status = FileUtil("somedir").rmdir()
self.assertTrue(status)
mock_rmdir.side_effect = OSError("fail")
status = FileUtil("somedir").rmdir()
self.assertFalse(status)
def get_ld_libdirs(self, force=False):
"""Get ld library paths"""
if force or not os.path.exists(self._container_ld_libdirs):
ld_list = self._find_ld_libdirs()
ld_str = ':'.join(ld_list)
FileUtil(self._container_ld_libdirs).putdata(ld_str, 'w')
return ld_list
ld_str = FileUtil(self._container_ld_libdirs).getdata('r')
return ld_str.split(':')
def get_mode(self):
"""Get execution mode"""
if self.force_mode:
return self.force_mode
futil_xm = FileUtil(self.container_execmode)
xmode = futil_xm.getdata('r').strip()
if not xmode:
xmode = Config.conf['default_execution_mode']
return xmode
def add(self, cont_path):
"""Add a container relative pathname as destination mountpoint"""
mountpoint = self.container_root + '/' + cont_path
orig_mpath = FileUtil(mountpoint).getvalid_path()
if orig_mpath:
self.mountpoints[cont_path] = \
orig_mpath.replace(self.container_root, "", 1)
if not self.mountpoints[cont_path]:
self.mountpoints[cont_path] = "/"
def test_17_remove(self, mock_regpre, mock_base, mock_absp, mock_safe,
mock_uid, mock_isdir, mock_isfile, mock_islink,
mock_remove, mock_msg, mock_exists, mock_realpath):
"""Test17 FileUtil.remove()."""
mock_msg.level = 0
mock_regpre.return_value = None
mock_base.return_value = '/filename4.txt'
mock_absp.return_value = '/filename4.txt'
mock_uid.return_value = 1000
# file does not exist (regression of #50)
mock_isdir.return_value = True
mock_isfile.return_value = True
mock_exists.return_value = True
mock_safe.return_value = True
mock_islink.return_value = False
mock_remove.return_value = None
Config().conf['uid'] = 1000
Config().conf['tmpdir'] = "/tmp"
mock_realpath.return_value = "/tmp"
# under /
futil = FileUtil("/filename4.txt")
status = futil.remove()
self.assertTrue(status)
# wrong uid
mock_base.return_value = 'filename4.txt'
mock_absp.return_value = '/tmp/filename4.txt'
mock_uid.return_value = 1001
futil = FileUtil("/tmp/filename4.txt")
status = futil.remove()
self.assertTrue(status)
# under /tmp TEST to be checked
# mock_base.return_value = 'filename4.txt'
# mock_absp.return_value = '/tmp/filename4.txt'
# mock_uid.return_value = 1000
# futil = FileUtil("/tmp/filename4.txt")
# status = futil.remove()
# self.assertTrue(status)
# under user home TEST to be checked
# mock_base.return_value = 'filename4.txt'
# mock_absp.return_value = '/home/user/.udocker/filename4.txt'
# futil = FileUtil("/home/user/.udocker/filename4.txt")
# futil.safe_prefixes.append("/home/user/.udocker")
# status = futil.remove()
# self.assertTrue(status)
# outside of scope 1
mock_base.return_value = 'filename4.txt'
mock_absp.return_value = '/etc/filename4.txt'
mock_safe.return_value = False
futil = FileUtil("/etc/filename4.txt")
futil.safe_prefixes = []
status = futil.remove()
self.assertTrue(status)
def test_05_mktmp(self, mock_regpre, mock_base, mock_absp):
"""Test05 FileUtil.mktmp()."""
mock_regpre.return_value = None
mock_base.return_value = 'filename2.txt'
mock_absp.return_value = '/tmp/filename2.txt'
Config().conf['tmpdir'] = '/somewhere'
tmp_file = FileUtil('filename2.txt').mktmp()
self.assertTrue(tmp_file.endswith('-filename2.txt'))
self.assertTrue(tmp_file.startswith('/somewhere/udocker-'))
self.assertGreater(len(tmp_file.strip()), 68)
def add_group(self, group, gid, users=None):
"""Add a group to a /etc/passwd file"""
users_str = ""
if isinstance(users, list):
for username in users:
users_str += "%s," % (username)
line = "%s:x:%s:%s\n" % (group, gid, users_str)
if line in FileUtil(self.group_file).getdata('r'):
return True
return FileUtil(self.group_file).putdata(line, 'a')
def test_31_find_exec(self, mock_regpre, mock_base, mock_absp,
mock_findexe):
"""Test31 FileUtil.find_exec()."""
mock_regpre.return_value = None
mock_base.return_value = 'ls'
mock_absp.return_value = '/bin/ls'
mock_findexe.return_value = '/bin/ls'
futil = FileUtil("/bin/ls")
status = futil.find_exec("/bin", "", "", ".", False)
self.assertEqual(status, "/bin/ls")
def test_29_cont2host(self, mock_regpre, mock_base, mock_absp, mock_c2h):
"""Test29 FileUtil.cont2host()."""
mock_regpre.return_value = None
mock_base.return_value = "somefile"
mock_absp.return_value = "somefile"
mock_c2h.return_value = "/ROOT/dir"
futil = FileUtil("somefile")
status = futil.cont2host("/ROOT/dir")
self.assertEqual(status, "/ROOT/dir")
self.assertTrue(mock_c2h.called)
def test_12_rchown(self, mock_regpre, mock_base, mock_absp, mock_fuchown):
"""Test12 FileUtil.rchown()."""
mock_regpre.return_value = None
mock_base.return_value = 'filename.txt'
mock_absp.return_value = '/tmp/filename.txt'
mock_fuchown.return_value = True
futil = FileUtil("somedir")
FileUtil.safe_prefixes = ["/tmp"]
status = futil.rchown()
self.assertTrue(status)
def test_15_rchmod(self, mock_regpre, mock_base, mock_absp, mock_fuchmod):
"""Test15 FileUtil.rchmod()."""
mock_regpre.return_value = None
mock_base.return_value = 'filename.txt'
mock_absp.return_value = '/tmp/filename.txt'
mock_fuchmod.return_value = True
futil = FileUtil("somedir")
FileUtil.safe_prefixes = ["/tmp"]
futil.rchmod()
self.assertTrue(mock_fuchmod.called)
def test_08_mktmpdir(self, mock_mkdir, mock_mktmp):
"""Test08 FileUtil.mktmpdir()."""
mock_mktmp.return_value = "/dir"
mock_mkdir.return_value = True
status = FileUtil("somedir").mktmpdir()
self.assertEqual(status, "/dir")
mock_mktmp.return_value = "/dir"
mock_mkdir.return_value = False
status = FileUtil("somedir").mktmpdir()
self.assertEqual(status, None)
def setup(self):
"""Prepare container for FileBind"""
if not os.path.isdir(self.container_orig_dir):
if not FileUtil(self.container_orig_dir).mkdir():
Msg().err("Error: creating dir:", self.container_orig_dir)
return False
if not os.path.isdir(self.container_bind_dir):
if not FileUtil(self.container_bind_dir).mkdir():
Msg().err("Error: creating dir:", self.container_bind_dir)
return False
return True
def test_42_match(self, mock_regpre, mock_base, mock_absp, mock_dirname,
mock_listdir, mock_isdir):
"""Test42 FileUtil.match()."""
mock_regpre.return_value = None
mock_base.return_value = "/con/filename.txt"
mock_absp.return_value = "/con/filename.txt"
mock_dirname.return_value = "/con/fil*"
mock_isdir.return_value = False
mock_listdir.return_value = list()
futil = FileUtil("/con/filename.txt")
status = futil.match()
self.assertEqual(status, [])
def test_23_size(self, mock_regpre, mock_base, mock_absp, mock_stat):
"""Test23 FileUtil.size()."""
mock_regpre.return_value = None
mock_base.return_value = 'filename.txt'
mock_absp.return_value = '/tmp/filename.txt'
mock_stat.return_value.st_size = 4321
size = FileUtil("somefile").size()
self.assertEqual(size, 4321)
mock_stat.side_effect = OSError("fail")
size = FileUtil("somefile").size()
self.assertEqual(size, -1)
def test_06_mkdir(self, mock_regpre, mock_base, mock_absp, mock_mkdirs):
"""Test06 FileUtil.mkdir()"""
mock_regpre.return_value = None
mock_base.return_value = 'filename.txt'
mock_absp.return_value = '/tmp/filename.txt'
mock_mkdirs.return_value = True
status = FileUtil("somedir").mkdir()
self.assertTrue(status)
mock_mkdirs.side_effect = OSError("fail")
status = FileUtil("somedir").mkdir()
self.assertFalse(status)
def test_01_init(self, mock_regpre, mock_base, mock_absp):
"""Test01 FileUtil() constructor."""
mock_regpre.return_value = None
mock_base.return_value = 'filename.txt'
mock_absp.return_value = '/tmp/filename.txt'
futil = FileUtil('filename.txt')
self.assertEqual(futil.filename, os.path.abspath('filename.txt'))
self.assertTrue(mock_regpre.called)
futil = FileUtil('-')
self.assertEqual(futil.filename, '-')
self.assertEqual(futil.basename, '-')
def test_20_copydir(self, mock_regpre, mock_base, mock_absp, mock_call):
"""Test20 FileUtil.copydir()."""
mock_regpre.return_value = None
mock_base.return_value = 'filename.txt'
mock_absp.return_value = '/tmp/filename.txt'
mock_call.return_value = 1
status = FileUtil("filename.txt").copydir("/dir1")
self.assertEqual(status, 1)
mock_call.return_value = 0
status = FileUtil("filename.txt").copydir("/dir1")
self.assertEqual(status, 0)
def test_22_isdir(self, mock_regpre, mock_base, mock_absp, mock_isdir):
"""Test22 FileUtil.isdir()."""
mock_regpre.return_value = None
mock_base.return_value = 'filename.txt'
mock_absp.return_value = '/tmp/filename.txt'
mock_isdir.return_value = True
status = FileUtil("filename.txt").isdir()
self.assertTrue(status)
mock_isdir.return_value = False
status = FileUtil("filename.txt").isdir()
self.assertFalse(status)
def del_imagerepo(self, imagerepo, tag, force=False):
"""Delete an image repository and its layers"""
tag_dir = self.cd_imagerepo(imagerepo, tag)
if (tag_dir and self._remove_layers(tag_dir, force)
and FileUtil(tag_dir).remove(recursive=True)):
self.cur_repodir = ""
self.cur_tagdir = ""
while imagerepo:
FileUtil(self.reposdir + '/' + imagerepo).rmdir()
imagerepo = "/".join(imagerepo.split("/")[:-1])
return True
return False
