def _load_spec(self, new=False): """Generate runc spec file""" if FileUtil(self._container_specfile).size() != -1 and new: FileUtil(self._container_specfile).register_prefix() FileUtil(self._container_specfile).remove() if FileUtil(self._container_specfile).size() == -1: cmd_l = [ self.executable, "spec", "--rootless", ] status = subprocess.call(cmd_l, shell=False, stderr=Msg.chlderr, close_fds=True, cwd=os.path.realpath(\ self._container_specdir)) if status: return False json_obj = None infile = None try: infile = open(self._container_specfile, 'r') json_obj = json.load(infile) except (IOError, OSError, AttributeError, ValueError, TypeError): json_obj = None if infile: infile.close() self._container_specjson = json_obj return json_obj
def save(self, imagetag_list, imagefile): """Save a set of image tags to a file similarly to docker save """ tmp_imagedir = FileUtil("save").mktmp() try: os.makedirs(tmp_imagedir) except (IOError, OSError): return False structure = dict() structure["manifest"] = [] structure["repositories"] = dict() status = False for (imagerepo, tag) in imagetag_list: status = self._save_image(imagerepo, tag, structure, tmp_imagedir) if not status: Msg().err("Error: save image failed:", imagerepo + ':' + tag) break if status: self.localrepo.save_json(tmp_imagedir + "/manifest.json", structure["manifest"]) self.localrepo.save_json(tmp_imagedir + "/repositories", structure["repositories"]) if not FileUtil(tmp_imagedir).tar(imagefile): Msg().err("Error: save image failed in writing tar", imagefile) status = False else: Msg().err("Error: no images specified") FileUtil(tmp_imagedir).remove(recursive=True) return status
def load(self, imagefile, imagerepo=None): """Generic load of image tags to a file""" if not os.path.exists(imagefile) and imagefile != '-': Msg().err("Error: image file does not exist:", imagefile) return False tmp_imagedir = FileUtil("load").mktmp() try: os.makedirs(tmp_imagedir) except (IOError, OSError): return False if not self._untar_saved_container(imagefile, tmp_imagedir): Msg().err("Error: failed to extract container:", imagefile) FileUtil(tmp_imagedir).remove(recursive=True) return False imagetype = self._get_imagedir_type(tmp_imagedir) if imagetype == "Docker": repositories = DockerLocalFileAPI(self.localrepo).load( tmp_imagedir, imagerepo) elif imagetype == "OCI": repositories = OciLocalFileAPI(self.localrepo).load( tmp_imagedir, imagerepo) else: repositories = [] FileUtil(tmp_imagedir).remove(recursive=True) return repositories
def test_16__removedir(self, mock_regpre, mock_base, mock_absp, mock_islink, mock_walk, mock_chmod, mock_unlink, mock_rmdir): """Test16 FileUtil._removedir().""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_walk.return_value = [ ("/tmp", ["dir"], ["file"]), ] mock_islink.side_effect = [False, True, True, True] mock_chmod.side_effect = [None, None, None, None] mock_unlink.side_effect = [None, None, None, None] mock_rmdir.side_effect = [None, None, None, None] # remove directory under /tmp OK futil = FileUtil("/tmp/directory") status = futil._removedir() self.assertTrue(mock_walk.called) self.assertTrue(mock_islink.call_count, 2) self.assertTrue(mock_chmod.call_count, 3) self.assertTrue(status) mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_walk.return_value = list() mock_chmod.side_effect = OSError("fail") futil = FileUtil("/directory") status = futil._removedir() self.assertFalse(status)
def select_singularity(self): """Set singularity executable and related variables""" self.executable = Config.conf['use_singularity_executable'] if self.executable != "UDOCKER" and not self.executable: self.executable = FileUtil("singularity").find_exec() if self.executable == "UDOCKER" or not self.executable: self.executable = "" arch = HostInfo().arch() image_list = [] if arch == "amd64": image_list = ["singularity-x86_64", "singularity"] elif arch == "i386": image_list = ["singularity-x86", "singularity"] elif arch == "arm64": image_list = ["singularity-arm64", "singularity"] elif arch == "arm": image_list = ["singularity-arm", "singularity"] f_util = FileUtil(self.localrepo.bindir) self.executable = f_util.find_file_in_dir(image_list) if not os.path.exists(self.executable): Msg().err("Error: singularity executable not found") sys.exit(1)
def _apply_whiteouts(self, tarf, destdir): """The layered filesystem of docker uses whiteout files to identify files or directories to be removed. The format is .wh.<filename> """ verbose = "" if Msg.level >= Msg.VER: verbose = 'v' Msg().out("Info: applying whiteouts:", tarf, l=Msg.INF) wildcards = [ "--wildcards", ] if not HostInfo().cmd_has_option("tar", wildcards[0]): wildcards = [] cmd = ["tar", "t" + verbose] + wildcards + ["-f", tarf, r"*/.wh.*"] whiteouts = Uprocess().get_output(cmd, True) if not whiteouts: return for wh_filename in whiteouts.split('\n'): if wh_filename: wh_basename = os.path.basename(wh_filename.strip()) wh_dirname = os.path.dirname(wh_filename) if wh_basename == ".wh..wh..opq": if not os.path.isdir(destdir + '/' + wh_dirname): continue for f_name in os.listdir(destdir + '/' + wh_dirname): rm_filename = destdir + '/' \ + wh_dirname + '/' + f_name FileUtil(rm_filename).remove(recursive=True) elif wh_basename.startswith(".wh."): rm_filename = destdir + '/' \ + wh_dirname + '/' \ + wh_basename.replace(".wh.", "", 1) FileUtil(rm_filename).remove(recursive=True) return
def set_version(self, version): """Set the version of the image TAG repository currently it supports Docker images with versions v1 and v2 to be invoked after setup_tag() """ if not (self.cur_repodir and self.cur_tagdir): return False if not os.path.exists(self.cur_repodir): return False if not os.path.exists(self.cur_tagdir): return False directory = self.cur_tagdir if (os.path.exists(directory + "/v1") and version != "v1" or os.path.exists(directory + "/v2") and version != "v2"): if len(os.listdir(directory)) == 1: try: FileUtil(directory + "/v1").remove() FileUtil(directory + "/v2").remove() except (IOError, OSError): pass if os.listdir(directory): return False try: # Create version file open(directory + "/" + version, 'a').close() except (IOError, OSError): return False return True
def add_user(self, user, passw, uid, gid, gecos, home, shell): """Add a *nix user to a /etc/passwd file""" line = "%s:%s:%s:%s:%s:%s:%s\n" % \ (user, passw, uid, gid, gecos, home, shell) if line in FileUtil(self.passwd_file).getdata('r'): return True return FileUtil(self.passwd_file).putdata(line, 'a')
def __init__(self, topdir=None): self.topdir = topdir if topdir else Config.conf['topdir'] self.bindir = Config.conf['bindir'] self.libdir = Config.conf['libdir'] self.docdir = Config.conf['docdir'] self.reposdir = Config.conf['reposdir'] self.layersdir = Config.conf['layersdir'] self.containersdir = Config.conf['containersdir'] self.homedir = Config.conf['homedir'] if not self.bindir: self.bindir = self.topdir + "/bin" if not self.libdir: self.libdir = self.topdir + "/lib" if not self.docdir: self.docdir = self.topdir + "/doc" if not self.reposdir: self.reposdir = self.topdir + "/repos" if not self.layersdir: self.layersdir = self.topdir + "/layers" if not self.containersdir: self.containersdir = self.topdir + "/containers" self.cur_repodir = "" self.cur_tagdir = "" self.cur_containerdir = "" FileUtil(self.reposdir).register_prefix() FileUtil(self.layersdir).register_prefix() FileUtil(self.containersdir).register_prefix()
def test_39__link_set(self, mock_regpre, mock_base, mock_absp, mock_lnchange, mock_readlink, mock_access): """Test39 FileUtil._link_set().""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_readlink.return_value = "X" futil = FileUtil("/con") status = futil._link_set("/con/lnk", "", "/con", False) self.assertFalse(status) mock_readlink.return_value = "/con" futil = FileUtil("/con") status = futil._link_set("/con/lnk", "", "/con", False) self.assertFalse(status) mock_readlink.return_value = "/HOST/DIR" futil = FileUtil("/con") status = futil._link_set("/con/lnk", "", "/con", False) self.assertTrue(mock_lnchange.called) self.assertTrue(status) mock_readlink.return_value = "/HOST/DIR" mock_access.return_value = True futil = FileUtil("/con") status = futil._link_set("/con/lnk", "", "/con", True) self.assertTrue(mock_lnchange.called) self.assertTrue(status) mock_readlink.return_value = "/HOST/DIR" mock_access.return_value = False futil = FileUtil("/con") status = futil._link_set("/con/lnk", "", "/con", True) self.assertTrue(mock_lnchange.called) self.assertTrue(status)
def test_13__chmod(self, mock_regpre, mock_base, mock_absp, mock_lstat, mock_chmod, mock_stat): """Test13 FileUtil._chmod().""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_lstat.return_value.st_mode = 33277 mock_chmod.return_value = None mock_stat.return_value.S_ISREG = True mock_stat.return_value.S_IMODE = 509 futil = FileUtil("somedir") futil._chmod("somefile") self.assertTrue(mock_lstat.called) self.assertTrue(mock_stat.S_ISREG.called) self.assertTrue(mock_stat.S_IMODE.called) mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_lstat.return_value.st_mode = 33277 mock_chmod.return_value = None mock_stat.return_value.S_ISREG = False mock_stat.return_value.S_ISDIR = True mock_stat.return_value.S_IMODE = 509 futil = FileUtil("somedir") futil._chmod("somefile") self.assertTrue(mock_chmod.called) self.assertTrue(mock_stat.S_IMODE.called)
def get_original_loader(self): """Get the pathname of the original ld.so""" if os.path.exists(self._container_ld_so_path): return FileUtil(self._container_ld_so_path).getdata('r').strip() elf_loader = self.guess_elf_loader() if elf_loader: FileUtil(self._container_ld_so_path).putdata(elf_loader, 'w') return elf_loader
def get_ld_libdirs(self, force=False): """Get ld library paths""" if force or not os.path.exists(self._container_ld_libdirs): ld_list = self._find_ld_libdirs() ld_str = ':'.join(ld_list) FileUtil(self._container_ld_libdirs).putdata(ld_str, 'w') return ld_list ld_str = FileUtil(self._container_ld_libdirs).getdata('r') return ld_str.split(':')
def test_07_rmdir(self, mock_rmdir): """Test07 FileUtil.rmdir().""" mock_rmdir.return_value = None status = FileUtil("somedir").rmdir() self.assertTrue(status) mock_rmdir.side_effect = OSError("fail") status = FileUtil("somedir").rmdir() self.assertFalse(status)
def test_17_remove(self, mock_regpre, mock_base, mock_absp, mock_safe, mock_uid, mock_isdir, mock_isfile, mock_islink, mock_remove, mock_msg, mock_exists, mock_realpath): """Test17 FileUtil.remove().""" mock_msg.level = 0 mock_regpre.return_value = None mock_base.return_value = '/filename4.txt' mock_absp.return_value = '/filename4.txt' mock_uid.return_value = 1000 # file does not exist (regression of #50) mock_isdir.return_value = True mock_isfile.return_value = True mock_exists.return_value = True mock_safe.return_value = True mock_islink.return_value = False mock_remove.return_value = None Config().conf['uid'] = 1000 Config().conf['tmpdir'] = "/tmp" mock_realpath.return_value = "/tmp" # under / futil = FileUtil("/filename4.txt") status = futil.remove() self.assertTrue(status) # wrong uid mock_base.return_value = 'filename4.txt' mock_absp.return_value = '/tmp/filename4.txt' mock_uid.return_value = 1001 futil = FileUtil("/tmp/filename4.txt") status = futil.remove() self.assertTrue(status) # under /tmp TEST to be checked # mock_base.return_value = 'filename4.txt' # mock_absp.return_value = '/tmp/filename4.txt' # mock_uid.return_value = 1000 # futil = FileUtil("/tmp/filename4.txt") # status = futil.remove() # self.assertTrue(status) # under user home TEST to be checked # mock_base.return_value = 'filename4.txt' # mock_absp.return_value = '/home/user/.udocker/filename4.txt' # futil = FileUtil("/home/user/.udocker/filename4.txt") # futil.safe_prefixes.append("/home/user/.udocker") # status = futil.remove() # self.assertTrue(status) # outside of scope 1 mock_base.return_value = 'filename4.txt' mock_absp.return_value = '/etc/filename4.txt' mock_safe.return_value = False futil = FileUtil("/etc/filename4.txt") futil.safe_prefixes = [] status = futil.remove() self.assertTrue(status)
def add_group(self, group, gid, users=None): """Add a group to a /etc/passwd file""" users_str = "" if isinstance(users, list): for username in users: users_str += "%s," % (username) line = "%s:x:%s:%s\n" % (group, gid, users_str) if line in FileUtil(self.group_file).getdata('r'): return True return FileUtil(self.group_file).putdata(line, 'a')
def test_08_mktmpdir(self, mock_mkdir, mock_mktmp): """Test08 FileUtil.mktmpdir().""" mock_mktmp.return_value = "/dir" mock_mkdir.return_value = True status = FileUtil("somedir").mktmpdir() self.assertEqual(status, "/dir") mock_mktmp.return_value = "/dir" mock_mkdir.return_value = False status = FileUtil("somedir").mktmpdir() self.assertEqual(status, None)
def setup(self): """Prepare container for FileBind""" if not os.path.isdir(self.container_orig_dir): if not FileUtil(self.container_orig_dir).mkdir(): Msg().err("Error: creating dir:", self.container_orig_dir) return False if not os.path.isdir(self.container_bind_dir): if not FileUtil(self.container_bind_dir).mkdir(): Msg().err("Error: creating dir:", self.container_bind_dir) return False return True
def test_01_init(self, mock_regpre, mock_base, mock_absp): """Test01 FileUtil() constructor.""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' futil = FileUtil('filename.txt') self.assertEqual(futil.filename, os.path.abspath('filename.txt')) self.assertTrue(mock_regpre.called) futil = FileUtil('-') self.assertEqual(futil.filename, '-') self.assertEqual(futil.basename, '-')
def test_06_mkdir(self, mock_regpre, mock_base, mock_absp, mock_mkdirs): """Test06 FileUtil.mkdir()""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_mkdirs.return_value = True status = FileUtil("somedir").mkdir() self.assertTrue(status) mock_mkdirs.side_effect = OSError("fail") status = FileUtil("somedir").mkdir() self.assertFalse(status)
def del_imagerepo(self, imagerepo, tag, force=False): """Delete an image repository and its layers""" tag_dir = self.cd_imagerepo(imagerepo, tag) if (tag_dir and self._remove_layers(tag_dir, force) and FileUtil(tag_dir).remove(recursive=True)): self.cur_repodir = "" self.cur_tagdir = "" while imagerepo: FileUtil(self.reposdir + '/' + imagerepo).rmdir() imagerepo = "/".join(imagerepo.split("/")[:-1]) return True return False
def test_20_copydir(self, mock_regpre, mock_base, mock_absp, mock_call): """Test20 FileUtil.copydir().""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_call.return_value = 1 status = FileUtil("filename.txt").copydir("/dir1") self.assertEqual(status, 1) mock_call.return_value = 0 status = FileUtil("filename.txt").copydir("/dir1") self.assertEqual(status, 0)
def test_22_isdir(self, mock_regpre, mock_base, mock_absp, mock_isdir): """Test22 FileUtil.isdir().""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_isdir.return_value = True status = FileUtil("filename.txt").isdir() self.assertTrue(status) mock_isdir.return_value = False status = FileUtil("filename.txt").isdir() self.assertFalse(status)
def test_23_size(self, mock_regpre, mock_base, mock_absp, mock_stat): """Test23 FileUtil.size().""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_stat.return_value.st_size = 4321 size = FileUtil("somefile").size() self.assertEqual(size, 4321) mock_stat.side_effect = OSError("fail") size = FileUtil("somefile").size() self.assertEqual(size, -1)
def test_26_putdata(self, mock_regpre, mock_base, mock_absp): """Test26 FileUtil.putdata()""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' futil = FileUtil("somefile") futil.filename = "" data = futil.putdata("qwerty") self.assertFalse(data) with patch(BUILTINS + '.open', mock_open()): data = FileUtil("somefile").putdata("qwerty") self.assertEqual(data, 'qwerty')
def _verify_keystore(self): """Verify the keystore file and directory""" keystore_uid = FileUtil(self.keystore_file).uid() if keystore_uid not in (-1, HostInfo.uid): raise IOError("not owner of keystore: %s" % (self.keystore_file)) keystore_dir = os.path.dirname(self.keystore_file) if FileUtil(keystore_dir).uid() != HostInfo.uid: raise IOError("keystore dir not found or not owner: %s" % (keystore_dir)) if (keystore_uid != -1 and (os.stat(self.keystore_file).st_mode & 0o077)): raise IOError("keystore is accessible to group or others: %s" % (self.keystore_file))
def test_36_copyto(self, mock_regpre, mock_base, mock_absp): """Test36 FileUtil.copyto().""" mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' with patch(BUILTINS + '.open', mock_open()): status = FileUtil("source").copyto("dest") self.assertTrue(status) status = FileUtil("source").copyto("dest", "w") self.assertTrue(status) status = FileUtil("source").copyto("dest", "a") self.assertTrue(status)
def test_19_tar(self, mock_regpre, mock_base, mock_absp, mock_call, mock_msg): """Test19 FileUtil.tar().""" mock_msg.level = 0 mock_msg.VER = 4 mock_regpre.return_value = None mock_base.return_value = 'filename.txt' mock_absp.return_value = '/tmp/filename.txt' mock_call.return_value = 1 status = FileUtil("tarball.tar").tar("tarball.tar") self.assertFalse(status) mock_call.return_value = 0 status = FileUtil("tarball.tar").tar("tarball.tar") self.assertTrue(status)
def _download(self, url): """Download a file """ download_file = FileUtil("udockertools").mktmp() if Msg.level <= Msg.DEF: Msg().setlevel(Msg.NIL) (hdr, dummy) = self.curl.get(url, ofile=download_file, follow=True) if Msg.level == Msg.NIL: Msg().setlevel() try: if "200" in hdr.data["X-ND-HTTPSTATUS"]: return download_file except (KeyError, TypeError, AttributeError): pass FileUtil(download_file).remove() return ""
def get_patch_last_time(self): """get time in seconds of last full patch of container""" last_time = FileUtil(self._container_patch_time).getdata('r').strip() try: return str(int(last_time)) except ValueError: return "0"