def __init__(self, topdir=None): self.topdir = topdir if topdir else Config.conf['topdir'] self.bindir = Config.conf['bindir'] self.libdir = Config.conf['libdir'] self.docdir = Config.conf['docdir'] self.reposdir = Config.conf['reposdir'] self.layersdir = Config.conf['layersdir'] self.containersdir = Config.conf['containersdir'] self.homedir = Config.conf['homedir'] if not self.bindir: self.bindir = self.topdir + "/bin" if not self.libdir: self.libdir = self.topdir + "/lib" if not self.docdir: self.docdir = self.topdir + "/doc" if not self.reposdir: self.reposdir = self.topdir + "/repos" if not self.layersdir: self.layersdir = self.topdir + "/layers" if not self.containersdir: self.containersdir = self.topdir + "/containers" self.cur_repodir = "" self.cur_tagdir = "" self.cur_containerdir = "" FileUtil(self.reposdir).register_prefix() FileUtil(self.layersdir).register_prefix() FileUtil(self.containersdir).register_prefix()
def patch_ld(self, output_elf=None): """Patch ld.so""" elf_loader = self.get_container_loader() if FileUtil(self._container_ld_so_orig).size() == -1: status = FileUtil(elf_loader).copyto(self._container_ld_so_orig) if not status: return False ld_data = FileUtil(self._container_ld_so_orig).getdata('rb') if not ld_data: ld_data = FileUtil(elf_loader).getdata('rb') if not ld_data: return False nul_etc = "\x00/\x00\x00\x00\x00\x00\x00\x00\x00\x00".encode() nul_lib = "\x00/\x00\x00\x00".encode() nul_usr = "******".encode() etc = "\x00/etc/ld.so".encode() lib = "\x00/lib".encode() usr = "******".encode() ld_data = ld_data.replace(etc, nul_etc).\ replace(lib, nul_lib).replace(usr, nul_usr) ld_library_path_orig = "\x00LD_LIBRARY_PATH\x00".encode() ld_library_path_new = "\x00LD_LIBRARY_REAL\x00".encode() ld_data = ld_data.replace(ld_library_path_orig, ld_library_path_new) if output_elf is None: return bool(FileUtil(elf_loader).putdata(ld_data, 'wb')) return bool(FileUtil(output_elf).putdata(ld_data, 'wb'))
def load(self, imagefile, imagerepo=None): """Generic load of image tags to a file""" if not os.path.exists(imagefile) and imagefile != '-': Msg().err("Error: image file does not exist:", imagefile) return False tmp_imagedir = FileUtil("load").mktmp() try: os.makedirs(tmp_imagedir) except (IOError, OSError): return False if not self._untar_saved_container(imagefile, tmp_imagedir): Msg().err("Error: failed to extract container:", imagefile) FileUtil(tmp_imagedir).remove(recursive=True) return False imagetype = self._get_imagedir_type(tmp_imagedir) if imagetype == "Docker": repositories = DockerLocalFileAPI(self.localrepo).load( tmp_imagedir, imagerepo) elif imagetype == "OCI": repositories = OciLocalFileAPI(self.localrepo).load( tmp_imagedir, imagerepo) else: repositories = [] FileUtil(tmp_imagedir).remove(recursive=True) return repositories
def set_version(self, version): """Set the version of the image TAG repository currently it supports Docker images with versions v1 and v2 to be invoked after setup_tag() """ if not (self.cur_repodir and self.cur_tagdir): return False if not os.path.exists(self.cur_repodir): return False if not os.path.exists(self.cur_tagdir): return False directory = self.cur_tagdir if (os.path.exists(directory + "/v1") and version != "v1" or os.path.exists(directory + "/v2") and version != "v2"): if len(os.listdir(directory)) == 1: try: FileUtil(directory + "/v1").remove() FileUtil(directory + "/v2").remove() except (IOError, OSError): pass if os.listdir(directory): return False try: # Create version file open(directory + "/" + version, 'a').close() except (IOError, OSError): return False return True
def add_user(self, user, passw, uid, gid, gecos, home, shell): """Add a *nix user to a /etc/passwd file""" line = "%s:%s:%s:%s:%s:%s:%s\n" % \ (user, passw, uid, gid, gecos, home, shell) if line in FileUtil(self.passwd_file).getdata('r'): return True return FileUtil(self.passwd_file).putdata(line, 'a')
def select_singularity(self): """Set singularity executable and related variables""" self.executable = Config.conf['use_singularity_executable'] if self.executable != "UDOCKER" and not self.executable: self.executable = FileUtil("singularity").find_exec() if self.executable == "UDOCKER" or not self.executable: self.executable = "" arch = HostInfo().arch() image_list = [] if arch == "amd64": image_list = ["singularity-x86_64", "singularity"] elif arch == "i386": image_list = ["singularity-x86", "singularity"] elif arch == "arm64": image_list = ["singularity-arm64", "singularity"] elif arch == "arm": image_list = ["singularity-arm", "singularity"] f_util = FileUtil(self.localrepo.bindir) self.executable = f_util.find_file_in_dir(image_list) if not os.path.exists(self.executable): Msg().err("Error: singularity executable not found") sys.exit(1)
def save(self, imagetag_list, imagefile): """Save a set of image tags to a file similarly to docker save """ tmp_imagedir = FileUtil("save").mktmp() try: os.makedirs(tmp_imagedir) except (IOError, OSError): return False structure = dict() structure["manifest"] = [] structure["repositories"] = dict() status = False for (imagerepo, tag) in imagetag_list: status = self._save_image(imagerepo, tag, structure, tmp_imagedir) if not status: Msg().err("Error: save image failed:", imagerepo + ':' + tag) break if status: self.localrepo.save_json(tmp_imagedir + "/manifest.json", structure["manifest"]) self.localrepo.save_json(tmp_imagedir + "/repositories", structure["repositories"]) if not FileUtil(tmp_imagedir).tar(imagefile): Msg().err("Error: save image failed in writing tar", imagefile) status = False else: Msg().err("Error: no images specified") FileUtil(tmp_imagedir).remove(recursive=True) return status
def _load_spec(self, new=False): """Generate runc spec file""" if FileUtil(self._container_specfile).size() != -1 and new: FileUtil(self._container_specfile).register_prefix() FileUtil(self._container_specfile).remove() if FileUtil(self._container_specfile).size() == -1: cmd_l = [ self.executable, "spec", "--rootless", ] status = subprocess.call(cmd_l, shell=False, stderr=Msg.chlderr, close_fds=True, cwd=os.path.realpath(\ self._container_specdir)) if status: return False json_obj = None infile = None try: infile = open(self._container_specfile, 'r') json_obj = json.load(infile) except (IOError, OSError, AttributeError, ValueError, TypeError): json_obj = None if infile: infile.close() self._container_specjson = json_obj return json_obj
def _apply_whiteouts(self, tarf, destdir): """The layered filesystem of docker uses whiteout files to identify files or directories to be removed. The format is .wh.<filename> """ verbose = "" if Msg.level >= Msg.VER: verbose = 'v' Msg().out("Info: applying whiteouts:", tarf, l=Msg.INF) wildcards = [ "--wildcards", ] if not HostInfo().cmd_has_option("tar", wildcards[0]): wildcards = [] cmd = ["tar", "t" + verbose] + wildcards + ["-f", tarf, r"*/.wh.*"] whiteouts = Uprocess().get_output(cmd, True) if not whiteouts: return for wh_filename in whiteouts.split('\n'): if wh_filename: wh_basename = os.path.basename(wh_filename.strip()) wh_dirname = os.path.dirname(wh_filename) if wh_basename == ".wh..wh..opq": if not os.path.isdir(destdir + '/' + wh_dirname): continue for f_name in os.listdir(destdir + '/' + wh_dirname): rm_filename = destdir + '/' \ + wh_dirname + '/' + f_name FileUtil(rm_filename).remove(recursive=True) elif wh_basename.startswith(".wh."): rm_filename = destdir + '/' \ + wh_dirname + '/' \ + wh_basename.replace(".wh.", "", 1) FileUtil(rm_filename).remove(recursive=True) return
def get_original_loader(self): """Get the pathname of the original ld.so""" if os.path.exists(self._container_ld_so_path): return FileUtil(self._container_ld_so_path).getdata('r').strip() elf_loader = self.guess_elf_loader() if elf_loader: FileUtil(self._container_ld_so_path).putdata(elf_loader, 'w') return elf_loader
def test_07_rmdir(self, mock_rmdir): """Test07 FileUtil.rmdir().""" mock_rmdir.return_value = None status = FileUtil("somedir").rmdir() self.assertTrue(status) mock_rmdir.side_effect = OSError("fail") status = FileUtil("somedir").rmdir() self.assertFalse(status)
def get_ld_libdirs(self, force=False): """Get ld library paths""" if force or not os.path.exists(self._container_ld_libdirs): ld_list = self._find_ld_libdirs() ld_str = ':'.join(ld_list) FileUtil(self._container_ld_libdirs).putdata(ld_str, 'w') return ld_list ld_str = FileUtil(self._container_ld_libdirs).getdata('r') return ld_str.split(':')
def get_mode(self): """Get execution mode""" if self.force_mode: return self.force_mode futil_xm = FileUtil(self.container_execmode) xmode = futil_xm.getdata('r').strip() if not xmode: xmode = Config.conf['default_execution_mode'] return xmode
def add(self, cont_path): """Add a container relative pathname as destination mountpoint""" mountpoint = self.container_root + '/' + cont_path orig_mpath = FileUtil(mountpoint).getvalid_path() if orig_mpath: self.mountpoints[cont_path] = \ orig_mpath.replace(self.container_root, "", 1) if not self.mountpoints[cont_path]: self.mountpoints[cont_path] = "/"
def test_17_remove(self, mock_regpre, mock_base, mock_absp, mock_safe, mock_uid, mock_isdir, mock_isfile, mock_islink, mock_remove, mock_msg, mock_exists, mock_realpath): """Test17 FileUtil.remove().""" mock_msg.level = 0 mock_regpre.return_value = None mock_base.return_value = '/filename4.txt' mock_absp.return_value = '/filename4.txt' mock_uid.return_value = 1000 # file does not exist (regression of #50) mock_isdir.return_value = True mock_isfile.return_value = True mock_exists.return_value = True mock_safe.return_value = True mock_islink.return_value = False mock_remove.return_value = None Config().conf['uid'] = 1000 Config().conf['tmpdir'] = "/tmp" mock_realpath.return_value = "/tmp" # under / futil = FileUtil("/filename4.txt") status = futil.remove() self.assertTrue(status) # wrong uid mock_base.return_value = 'filename4.txt' mock_absp.return_value = '/tmp/filename4.txt' mock_uid.return_value = 1001 futil = FileUtil("/tmp/filename4.txt") status = futil.remove() self.assertTrue(status) # under /tmp TEST to be checked # mock_base.return_value = 'filename4.txt' # mock_absp.return_value = '/tmp/filename4.txt' # mock_uid.return_value = 1000 # futil = FileUtil("/tmp/filename4.txt") # status = futil.remove() # self.assertTrue(status) # under user home TEST to be checked # mock_base.return_value = 'filename4.txt' # mock_absp.return_value = '/home/user/.udocker/filename4.txt' # futil = FileUtil("/home/user/.udocker/filename4.txt") # futil.safe_prefixes.append("/home/user/.udocker") # status = futil.remove() # self.assertTrue(status) # outside of scope 1 mock_base.return_value = 'filename4.txt' mock_absp.return_value = '/etc/filename4.txt' mock_safe.return_value = False futil = FileUtil("/etc/filename4.txt") futil.safe_prefixes = [] status = futil.remove() self.assertTrue(status)
def test_05_mktmp(self, mock_regpre, mock_base, mock_absp): """Test05 FileUtil.mktmp().""" mock_regpre.return_value = None mock_base.return_value = 'filename2.txt' mock_absp.return_value = '/tmp/filename2.txt' Config().conf['tmpdir'] = '/somewhere' tmp_file = FileUtil('filename2.txt').mktmp() self.assertTrue(tmp_file.endswith('-filename2.txt')) self.assertTrue(tmp_file.startswith('/somewhere/udocker-')) self.assertGreater(len(tmp_file.strip()), 68)
def add_group(self, group, gid, users=None): """Add a group to a /etc/passwd file""" users_str = "" if isinstance(users, list): for username in users: users_str += "%s," % (username) line = "%s:x:%s:%s\n" % (group, gid, users_str) if line in FileUtil(self.group_file).getdata('r'): return True return FileUtil(self.group_file).putdata(line, 'a')
def test_31_find_exec(self, mock_regpre, mock_base, mock_absp, mock_findexe): """Test31 FileUtil.find_exec().""" mock_regpre.return_value = None mock_base.return_value = 'ls' mock_absp.return_value = '/bin/ls' mock_findexe.return_value = '/bin/ls' futil = FileUtil("/bin/ls") status = futil.find_exec("/bin", "", "", ".", False) self.assertEqual(status, "/bin/ls")
def test_29_cont2host(self, mock_regpre, mock_base, mock_absp, mock_c2h): """Test29 FileUtil.cont2host().""" mock_regpre.return_value = None mock_base.return_value = "somefile" mock_absp.return_value = "somefile" mock_c2h.return_value = "/ROOT/dir" futil = FileUtil("somefile") status = futil.cont2host("/ROOT/dir") self.assertEqual(status, "/ROOT/dir") self.assertTrue(mock_c2h.called)
def test_12_rchown(self, mock_regpre, mock_base, mock_absp, mock_fuchown): """Test12 FileUtil.rchown().""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_fuchown.return_value = True futil = FileUtil("somedir") FileUtil.safe_prefixes = ["/tmp"] status = futil.rchown() self.assertTrue(status)
def test_15_rchmod(self, mock_regpre, mock_base, mock_absp, mock_fuchmod): """Test15 FileUtil.rchmod().""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_fuchmod.return_value = True futil = FileUtil("somedir") FileUtil.safe_prefixes = ["/tmp"] futil.rchmod() self.assertTrue(mock_fuchmod.called)
def test_08_mktmpdir(self, mock_mkdir, mock_mktmp): """Test08 FileUtil.mktmpdir().""" mock_mktmp.return_value = "/dir" mock_mkdir.return_value = True status = FileUtil("somedir").mktmpdir() self.assertEqual(status, "/dir") mock_mktmp.return_value = "/dir" mock_mkdir.return_value = False status = FileUtil("somedir").mktmpdir() self.assertEqual(status, None)
def setup(self): """Prepare container for FileBind""" if not os.path.isdir(self.container_orig_dir): if not FileUtil(self.container_orig_dir).mkdir(): Msg().err("Error: creating dir:", self.container_orig_dir) return False if not os.path.isdir(self.container_bind_dir): if not FileUtil(self.container_bind_dir).mkdir(): Msg().err("Error: creating dir:", self.container_bind_dir) return False return True
def test_42_match(self, mock_regpre, mock_base, mock_absp, mock_dirname, mock_listdir, mock_isdir): """Test42 FileUtil.match().""" mock_regpre.return_value = None mock_base.return_value = "/con/filename.txt" mock_absp.return_value = "/con/filename.txt" mock_dirname.return_value = "/con/fil*" mock_isdir.return_value = False mock_listdir.return_value = list() futil = FileUtil("/con/filename.txt") status = futil.match() self.assertEqual(status, [])
def test_23_size(self, mock_regpre, mock_base, mock_absp, mock_stat): """Test23 FileUtil.size().""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_stat.return_value.st_size = 4321 size = FileUtil("somefile").size() self.assertEqual(size, 4321) mock_stat.side_effect = OSError("fail") size = FileUtil("somefile").size() self.assertEqual(size, -1)
def test_06_mkdir(self, mock_regpre, mock_base, mock_absp, mock_mkdirs): """Test06 FileUtil.mkdir()""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_mkdirs.return_value = True status = FileUtil("somedir").mkdir() self.assertTrue(status) mock_mkdirs.side_effect = OSError("fail") status = FileUtil("somedir").mkdir() self.assertFalse(status)
def test_01_init(self, mock_regpre, mock_base, mock_absp): """Test01 FileUtil() constructor.""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' futil = FileUtil('filename.txt') self.assertEqual(futil.filename, os.path.abspath('filename.txt')) self.assertTrue(mock_regpre.called) futil = FileUtil('-') self.assertEqual(futil.filename, '-') self.assertEqual(futil.basename, '-')
def test_20_copydir(self, mock_regpre, mock_base, mock_absp, mock_call): """Test20 FileUtil.copydir().""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_call.return_value = 1 status = FileUtil("filename.txt").copydir("/dir1") self.assertEqual(status, 1) mock_call.return_value = 0 status = FileUtil("filename.txt").copydir("/dir1") self.assertEqual(status, 0)
def test_22_isdir(self, mock_regpre, mock_base, mock_absp, mock_isdir): """Test22 FileUtil.isdir().""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_isdir.return_value = True status = FileUtil("filename.txt").isdir() self.assertTrue(status) mock_isdir.return_value = False status = FileUtil("filename.txt").isdir() self.assertFalse(status)
def del_imagerepo(self, imagerepo, tag, force=False): """Delete an image repository and its layers""" tag_dir = self.cd_imagerepo(imagerepo, tag) if (tag_dir and self._remove_layers(tag_dir, force) and FileUtil(tag_dir).remove(recursive=True)): self.cur_repodir = "" self.cur_tagdir = "" while imagerepo: FileUtil(self.reposdir + '/' + imagerepo).rmdir() imagerepo = "/".join(imagerepo.split("/")[:-1]) return True return False