def test_it(self): orig = contentsSet([ fs.fsFile('/cheddar', strict=False), fs.fsFile('/sporks-suck', strict=False), fs.fsDir('/foons-rule', strict=False), fs.fsDir('/mango', strict=False) ]) engine = fake_engine(mode=const.INSTALL_MODE) def run(func): new = contentsSet(orig) self.kls(func)(engine, {'new_cset':new}) return new self.assertEqual(orig, run(lambda s:False)) self.assertEqual([], run(post_curry(isinstance, fs.fsDir)).dirs()) self.assertEqual(sorted(orig.files()), sorted(run(post_curry(isinstance, fs.fsDir)).dirs(True))) # check noisyness. info = [] engine = fake_engine(observer=make_fake_reporter(info=info.append), mode=const.REPLACE_MODE) run(lambda s:False) self.assertFalse(info) run(post_curry(isinstance, fs.fsDir)) self.assertEqual(len(info), 2) # ensure only the relevant files show. self.assertNotIn('/cheddar', ' '.join(info)) self.assertNotIn('/sporks-suck', ' '.join(info)) self.assertIn('/foons-rule', ' '.join(info)) self.assertIn('/mango', ' '.join(info))
def test_it(self): orig = contentsSet([ fs.fsFile('/cheddar', strict=False), fs.fsFile('/sporks-suck', strict=False), fs.fsDir('/foons-rule', strict=False), fs.fsDir('/mango', strict=False) ]) engine = fake_engine(mode=const.INSTALL_MODE) def run(func): new = contentsSet(orig) self.kls(func)(engine, {'new_cset':new}) return new self.assertEqual(orig, run(lambda s:False)) self.assertEqual([], run(post_curry(isinstance, fs.fsDir)).dirs()) self.assertEqual(orig.files(), run(post_curry(isinstance, fs.fsDir)).dirs(True)) # check noisyness. info = [] engine = fake_engine(observer=fake_reporter(info=info.append), mode=const.REPLACE_MODE) run(lambda s:False) self.assertFalse(info) run(post_curry(isinstance, fs.fsDir)) self.assertEqual(len(info), 2) # ensure only the relevant files show. self.assertNotIn('/cheddar', ' '.join(info)) self.assertNotIn('/sporks-suck', ' '.join(info)) self.assertIn('/foons-rule', ' '.join(info)) self.assertIn('/mango', ' '.join(info))
def test_is_funcs(self): # verify it intercepts the missing attr self.assertFalse(fs.isdir(object())) self.assertFalse(fs.isreg(object())) self.assertFalse(fs.isfifo(object())) self.assertTrue(fs.isdir(fs.fsDir('/tmp', strict=False))) self.assertFalse(fs.isreg(fs.fsDir('/tmp', strict=False))) self.assertTrue(fs.isreg(fs.fsFile('/tmp', strict=False)))
def test_intersect(self): open(pjoin(self.dir, 'reg'), 'w').close() cset = contentsSet([fs.fsFile('reg', strict=False)]) cset = cset.insert_offset(self.dir) self.assertEqual(contentsSet(livefs.intersect(cset)), cset) cset = contentsSet([fs.fsFile('reg/foon', strict=False), fs.fsFile('reg/dar', strict=False), fs.fsDir('reg/dir', strict=False)]).insert_offset(self.dir) self.assertEqual(list(livefs.intersect(cset)), []) cset = contentsSet([fs.fsDir('reg', strict=False)]) self.assertEqual(list(livefs.intersect(cset)), [])
def test_it(self): o = fs.fsDir(pjoin(self.dir, "mkdir_test"), strict=False) self.assertTrue(ops.default_mkdir(o)) old_umask = os.umask(0) try: self.assertEqual((os.stat(o.location).st_mode & 04777), 0777 & ~old_umask) finally: os.umask(old_umask) os.rmdir(o.location) o = fs.fsDir(pjoin(self.dir, "mkdir_test2"), strict=False, mode=0750) self.assertTrue(ops.default_mkdir(o)) self.assertEqual(os.stat(o.location).st_mode & 04777, 0750)
def test_intersect(self): open(pjoin(self.dir, 'reg'), 'w').close() cset = contentsSet([fs.fsFile('reg', strict=False)]) cset = cset.insert_offset(self.dir) self.assertEqual(contentsSet(livefs.intersect(cset)), cset) cset = contentsSet([ fs.fsFile('reg/foon', strict=False), fs.fsFile('reg/dar', strict=False), fs.fsDir('reg/dir', strict=False) ]).insert_offset(self.dir) self.assertEqual(list(livefs.intersect(cset)), []) cset = contentsSet([fs.fsDir('reg', strict=False)]) self.assertEqual(list(livefs.intersect(cset)), [])
def archive_to_fsobj(src_tar): psep = os.path.sep for member in src_tar: d = { "uid": member.uid, "gid": member.gid, "mtime": member.mtime, "mode": member.mode } location = psep + member.name.strip(psep) if member.isdir(): if member.name.strip(psep) == ".": continue yield fsDir(location, **d) elif member.isreg(): d["data"] = invokable_data_source.wrap_function( partial(src_tar.extractfile, member.name), returns_text=False, returns_handle=True) # suppress hardlinks until the rest of pkgcore is updated for it. d["dev"] = None d["inode"] = None yield fsFile(location, **d) elif member.issym() or member.islnk(): yield fsSymlink(location, member.linkname, **d) elif member.isfifo(): yield fsFifo(location, **d) elif member.isdev(): d["major"] = long(member.major) d["minor"] = long(member.minor) yield fsDev(location, **d) else: raise AssertionError( "unknown type %r, %r was encounted walking tarmembers" % (member, member.type))
def archive_to_fsobj(src_tar): psep = os.path.sep for member in src_tar: d = { "uid":member.uid, "gid":member.gid, "mtime":member.mtime, "mode":member.mode} location = psep + member.name.strip(psep) if member.isdir(): if member.name.strip(psep) == ".": continue yield fsDir(location, **d) elif member.isreg(): d["data"] = invokable_data_source.wrap_function(partial( src_tar.extractfile, member.name), returns_text=False, returns_handle=True) # suppress hardlinks until the rest of pkgcore is updated for it. d["dev"] = None d["inode"] = None yield fsFile(location, **d) elif member.issym() or member.islnk(): yield fsSymlink(location, member.linkname, **d) elif member.isfifo(): yield fsFifo(location, **d) elif member.isdev(): d["major"] = long(member.major) d["minor"] = long(member.minor) yield fsDev(location, **d) else: raise AssertionError( "unknown type %r, %r was encounted walking tarmembers" % (member, member.type))
def merge_contents(cset, offset=None, callback=None): """ merge a :class:`pkgcore.fs.contents.contentsSet` instance to the livefs :param cset: :class:`pkgcore.fs.contents.contentsSet` instance :param offset: if not None, offset to prefix all locations with. Think of it as target dir. :param callback: callable to report each entry being merged; given a single arg, the fs object being merged. :raise EnvironmentError: Thrown for permission failures. """ if callback is None: callback = lambda obj:None ensure_perms = get_plugin("fs_ops.ensure_perms") copyfile = get_plugin("fs_ops.copyfile") mkdir = get_plugin("fs_ops.mkdir") if not isinstance(cset, contents.contentsSet): raise TypeError("cset must be a contentsSet, got %r" % (cset,)) if offset is not None: if os.path.exists(offset): if not os.path.isdir(offset): raise TypeError("offset must be a dir, or not exist: %s" % offset) else: mkdir(fs.fsDir(offset, strict=False)) iterate = partial(contents.offset_rewriter, offset.rstrip(os.path.sep)) else: iterate = iter d = list(iterate(cset.iterdirs())) d.sort() for x in d: callback(x) try: # we pass in the stat ourselves, using stat instead of # lstat gen_obj uses internally; this is the equivalent of # "deference that link" obj = gen_obj(x.location, stat=os.stat(x.location)) if not fs.isdir(obj): raise Exception( "%s exists and needs to be a dir, but is a %s" % (x.location, obj)) ensure_perms(x, obj) except OSError, oe: if oe.errno != errno.ENOENT: raise try: # we do this form to catch dangling symlinks mkdir(x) except OSError, oe: if oe.errno != errno.EEXIST: raise os.unlink(x.location) mkdir(x) ensure_perms(x)
def _iter_contents(self): self.clear() for line in self._get_fd(): if not line: continue s = line.split(" ") if s[0] in ("dir", "dev", "fif"): path = ' '.join(s[1:]) if s[0] == 'dir': obj = fs.fsDir(path, strict=False) elif s[0] == 'dev': obj = LookupFsDev(path, strict=False) else: obj = fs.fsFifo(path, strict=False) elif s[0] == "obj": path = ' '.join(s[1:-2]) obj = fs.fsFile( path, chksums={"md5":int(s[-2], 16)}, mtime=int(s[-1]), strict=False) elif s[0] == "sym": try: p = s.index("->") obj = fs.fsLink(' '.join(s[1:p]), ' '.join(s[p+1:-1]), mtime=int(s[-1]), strict=False) except ValueError: # XXX throw a corruption error raise else: raise Exception(f"unknown entry type {line!r}") yield obj
def _iter_contents(self): self.clear() for line in self._get_fd(): if not line: continue s = line.split(" ") if s[0] in ("dir", "dev", "fif"): path = ' '.join(s[1:]) if s[0] == 'dir': obj = fs.fsDir(path, strict=False) elif s[0] == 'dev': obj = LookupFsDev(path, strict=False) else: obj = fs.fsFifo(path, strict=False) elif s[0] == "obj": path = ' '.join(s[1:-2]) obj = fs.fsFile( path, chksums={"md5":long(s[-2], 16)}, mtime=long(s[-1]), strict=False) elif s[0] == "sym": try: p = s.index("->") obj = fs.fsLink(' '.join(s[1:p]), ' '.join(s[p+1:-1]), mtime=long(s[-1]), strict=False) except ValueError: # XXX throw a corruption error raise else: raise Exception( "unknown entry type %r" % (line,)) yield obj
def test_dir_over_file(self): # according to the spec, dirs can't be merged over files that # aren't dirs or symlinks to dirs path = pjoin(self.dir, "file2dir") open(path, 'w').close() d = fs.fsDir(path, mode=0o755, mtime=0, uid=os.getuid(), gid=os.getgid()) cset = contents.contentsSet([d]) self.assertRaises(ops.CannotOverwrite, ops.merge_contents, cset)
def test_trigger(self): self.assertContents() self.assertContents([fs.fsFile("/foon", mode=0o644, uid=2, gid=1, strict=False)]) self.assertContents([fs.fsFile("/foon", mode=0o646, uid=1, gid=1, strict=False)]) self.assertContents([fs.fsFile("/foon", mode=0o4766, uid=1, gid=2, strict=False)]) self.assertContents([fs.fsFile("/blarn", mode=0o2700, uid=2, gid=2, strict=False), fs.fsDir("/dir", mode=0o500, uid=2, gid=2, strict=False)]) self.assertContents([fs.fsFile("/blarn", mode=0o2776, uid=2, gid=2, strict=False), fs.fsDir("/dir", mode=0o2777, uid=1, gid=2, strict=False)]) self.assertContents([fs.fsFile("/blarn", mode=0o6772, uid=2, gid=2, strict=False), fs.fsDir("/dir", mode=0o4774, uid=1, gid=1, strict=False)])
def gen_obj(path, stat=None, chksum_handlers=None, real_location=None, stat_func=os.lstat, **overrides): """ given a fs path, and an optional stat, create an appropriate fs obj. :param stat: stat object to reuse if available :param real_location: real path to the object if path is the desired location, rather then existent location. :raise KeyError: if no obj type matches the stat checks :return: :obj:`pkgcore.fs.fs.fsBase` derivative """ if real_location is None: real_location = path if stat is None: try: stat = stat_func(real_location) except EnvironmentError as e: if stat_func == os.lstat or e.errno != errno.ENOENT: raise stat = os.lstat(real_location) mode = stat.st_mode d = { "mtime": stat.st_mtime, "mode": S_IMODE(mode), "uid": stat.st_uid, "gid": stat.st_gid } if S_ISREG(mode): d["size"] = stat.st_size d["data"] = local_source(real_location) d["dev"] = stat.st_dev d["inode"] = stat.st_ino if chksum_handlers is not None: d["chf_types"] = chksum_handlers d.update(overrides) return fsFile(path, **d) d.update(overrides) if S_ISDIR(mode): return fsDir(path, **d) elif S_ISLNK(mode): d["target"] = os.readlink(real_location) return fsSymlink(path, **d) elif S_ISFIFO(mode): return fsFifo(path, **d) else: major, minor = get_major_minor(stat) d["minor"] = minor d["major"] = major d["mode"] = mode return fsDev(path, **d)
def archive_to_fsobj(src_tar): psep = os.path.sep dev = _unique_inode() # inode cache used for supporting hardlinks. # Since the tarfile specifies a hardlink target by path (rather than internally # consistent inode numbers), we have to normalize the path lookup into this cache # via abspath(os.path.join('/', key))... inodes = {} for member in src_tar: d = { "uid": member.uid, "gid": member.gid, "mtime": member.mtime, "mode": member.mode } location = os.path.abspath(os.path.join(psep, member.name.strip(psep))) if member.isdir(): if member.name.strip(psep) == ".": continue yield fsDir(location, **d) elif member.isreg() or member.islnk(): d["dev"] = dev if member.islnk(): target = os.path.abspath(os.path.join(psep, member.linkname)) inode = inodes.get(target) if inode is None: raise AssertionError( "Tarfile file %r is a hardlink to %r, but we can't " "find the resolved hardlink target %r in the archive. " "This means either a bug in pkgcore, or a malformed " "tarball." % (member.name, member.linkname, target)) d["inode"] = inode else: d["inode"] = inode = _unique_inode() # Add the new file to the inode cache even if we're currently processing a # hardlink; tar allows for hardlink chains of x -> y -> z; thus we have # to ensure 'y' is in the cache alongside it's target z to support 'x' # later lookup. inodes[location] = inode d["data"] = invokable_data_source.wrap_function( partial(src_tar.extractfile, member.name), returns_text=False, returns_handle=True) yield fsFile(location, **d) elif member.issym() or member.islnk(): yield fsSymlink(location, member.linkname, **d) elif member.isfifo(): yield fsFifo(location, **d) elif member.isdev(): d["major"] = int(member.major) d["minor"] = int(member.minor) yield fsDev(location, **d) else: raise AssertionError( "unknown type %r, %r was encounted walking tarmembers" % (member, member.type))
def archive_to_fsobj(src_tar): psep = os.path.sep dev = _unique_inode() # inode cache used for supporting hardlinks. # Since the tarfile specifies a hardlink target by path (rather than internally # consistent inode numbers), we have to normalize the path lookup into this cache # via abspath(os.path.join('/', key))... inodes = {} for member in src_tar: d = { "uid":member.uid, "gid":member.gid, "mtime":member.mtime, "mode":member.mode} location = os.path.abspath(os.path.join(psep, member.name.strip(psep))) if member.isdir(): if member.name.strip(psep) == ".": continue yield fsDir(location, **d) elif member.isreg() or member.islnk(): d["dev"] = dev if member.islnk(): target = os.path.abspath(os.path.join(psep, member.linkname)) inode = inodes.get(target) if inode is None: raise AssertionError( "Tarfile file %r is a hardlink to %r, but we can't " "find the resolved hardlink target %r in the archive. " "This means either a bug in pkgcore, or a malformed " "tarball." % (member.name, member.linkname, target)) d["inode"] = inode else: d["inode"] = inode = _unique_inode() # Add the new file to the inode cache even if we're currently processing a # hardlink; tar allows for hardlink chains of x -> y -> z; thus we have # to ensure 'y' is in the cache alongside it's target z to support 'x' # later lookup. inodes[location] = inode d["data"] = invokable_data_source.wrap_function(partial( src_tar.extractfile, member.name), returns_text=False, returns_handle=True) yield fsFile(location, **d) elif member.issym() or member.islnk(): yield fsSymlink(location, member.linkname, **d) elif member.isfifo(): yield fsFifo(location, **d) elif member.isdev(): d["major"] = int(member.major) d["minor"] = int(member.minor) yield fsDev(location, **d) else: raise AssertionError( "unknown type %r, %r was encounted walking tarmembers" % (member, member.type))
def gen_obj(path, stat=None, chksum_handlers=None, real_location=None, stat_func=os.lstat, **overrides): """ given a fs path, and an optional stat, create an appropriate fs obj. :param stat: stat object to reuse if available :param real_location: real path to the object if path is the desired location, rather then existent location. :raise KeyError: if no obj type matches the stat checks :return: :obj:`pkgcore.fs.fs.fsBase` derivative """ if real_location is None: real_location = path if stat is None: try: stat = stat_func(real_location) except EnvironmentError as e: if stat_func == os.lstat or e.errno != errno.ENOENT: raise stat = os.lstat(real_location) mode = stat.st_mode d = {"mtime":stat.st_mtime, "mode":S_IMODE(mode), "uid":stat.st_uid, "gid":stat.st_gid} if S_ISREG(mode): d["size"] = stat.st_size d["data"] = local_source(real_location) d["dev"] = stat.st_dev d["inode"] = stat.st_ino if chksum_handlers is not None: d["chf_types"] = chksum_handlers d.update(overrides) return fsFile(path, **d) d.update(overrides) if S_ISDIR(mode): return fsDir(path, **d) elif S_ISLNK(mode): d["target"] = os.readlink(real_location) return fsSymlink(path, **d) elif S_ISFIFO(mode): return fsFifo(path, **d) else: major, minor = get_major_minor(stat) d["minor"] = minor d["major"] = major d["mode"] = mode return fsDev(path, **d)
def add_missing_directories(self, mode=0o775, uid=0, gid=0, mtime=None): """Ensure that a directory node exists for each path; add if missing.""" missing = (x.dirname for x in self) missing = set(x for x in missing if x not in self) if mtime is None: mtime = time.time() # have to go recursive since many directories may be missing. missing_initial = list(missing) for x in missing_initial: target = path.dirname(x) while target not in missing and target not in self: missing.add(target) target = path.dirname(target) missing.discard("/") self.update(fs.fsDir(location=x, mode=mode, uid=uid, gid=gid, mtime=mtime) for x in missing)
def add_missing_directories(self, mode=0o775, uid=0, gid=0, mtime=None): """Ensure that a directory node exists for each path; add if missing.""" missing = (x.dirname for x in self) missing = set(x for x in missing if x not in self) if mtime is None: mtime = time.time() # have to go recursive since many directories may be missing. missing_initial = list(missing) for x in missing_initial: target = path.dirname(x) while target not in missing and target not in self: missing.add(target) target = path.dirname(target) missing.discard("/") self.update( fs.fsDir(location=x, mode=mode, uid=uid, gid=gid, mtime=mtime) for x in missing)
def test_puke_on_dirs(self): path = pjoin(self.dir, "puke_dir") self.assertRaises(TypeError, ops.default_copyfile, fs.fsDir(path, strict=False)) os.mkdir(path) fp = pjoin(self.dir, "foon") open(fp, "w").close() f = livefs.gen_obj(fp) self.assertRaises(TypeError, livefs.gen_obj(fp).change_attributes(location=path)) # test sym over a directory. f = fs.fsSymlink(path, fp, mode=0644, mtime=0, uid=os.getuid(), gid=os.getgid()) self.assertRaises(TypeError, ops.default_copyfile, f) os.unlink(fp) os.mkdir(fp) self.assertRaises(ops.CannotOverwrite, ops.default_copyfile, f)
def check_set_op(self, name, ret, source=None): if source is None: source = [[fs.fsDir("/tmp", strict=False)], [fs.fsFile("/tmp", strict=False)]] c1, c2 = [contents.contentsSet(x) for x in source] if name.endswith("_update"): getattr(c1, name)(c2) c3 = c1 else: c3 = getattr(c1, name)(c2) self.assertEqual(set(ret), set(x.location for x in c3)) c1, c2 = [contents.contentsSet(x) for x in source] if name.endswith("_update"): getattr(c1, name)(iter(c2)) c3 = c1 else: c3 = getattr(c1, name)(iter(c2)) self.assertEqual(set(ret), set(x.location for x in c3))
def check_set_op(self, name, ret, source=None): if source is None: source = [[fs.fsDir("/tmp", strict=False)], [fs.fsFile("/tmp", strict=False)]] c1, c2 = [contents.contentsSet(x) for x in source] if name.endswith("_update"): getattr(c1, name)(c2) c3 = c1 else: c3 = getattr(c1, name)(c2) self.assertEqual( set(ret), set(x.location for x in c3)) c1, c2 = [contents.contentsSet(x) for x in source] if name.endswith("_update"): getattr(c1, name)(iter(c2)) c3 = c1 else: c3 = getattr(c1, name)(iter(c2)) self.assertEqual( set(ret), set(x.location for x in c3))
def add_missing_directories(self, mode=0775, uid=0, gid=0, mtime=None): """Ensure that a directory node exists for each path; add if missing.""" missing = (x.dirname for x in self) missing = set(x for x in missing if x not in self) if mtime is None: mtime = time.time() # have to go recursive since many directories may be missing. missing_initial = list(missing) for x in missing_initial: target = path.dirname(x) while target not in missing and target not in self: missing.add(target) target = path.dirname(target) missing.discard("/") self.update( fs.fsDir(location=x, mode=mode, uid=uid, gid=gid, mtime=mtime) for x in missing) class OrderedContentsSet(contentsSet): def __init__(self, initial=None, mutable=False, add_missing_directories=False): contentsSet.__init__(self, mutable=True) self._dict = OrderedDict() if initial: self.update(initial) # some sources are a bit stupid, tarballs for example. # add missing directories if requested if add_missing_directories:
def merge_contents(cset, offset=None, callback=None): """ merge a :class:`pkgcore.fs.contents.contentsSet` instance to the livefs :param cset: :class:`pkgcore.fs.contents.contentsSet` instance :param offset: if not None, offset to prefix all locations with. Think of it as target dir. :param callback: callable to report each entry being merged; given a single arg, the fs object being merged. :raise EnvironmentError: Thrown for permission failures. """ if callback is None: callback = lambda obj: None ensure_perms = get_plugin("fs_ops.ensure_perms") copyfile = get_plugin("fs_ops.copyfile") mkdir = get_plugin("fs_ops.mkdir") if not isinstance(cset, contents.contentsSet): raise TypeError("cset must be a contentsSet, got %r" % (cset, )) if offset is not None: if os.path.exists(offset): if not os.path.isdir(offset): raise TypeError("offset must be a dir, or not exist: %s" % offset) else: mkdir(fs.fsDir(offset, strict=False)) iterate = partial(contents.offset_rewriter, offset.rstrip(os.path.sep)) else: iterate = iter d = list(iterate(cset.iterdirs())) d.sort() for x in d: callback(x) try: # we pass in the stat ourselves, using stat instead of # lstat gen_obj uses internally; this is the equivalent of # "deference that link" obj = gen_obj(x.location, stat=os.stat(x.location)) if not fs.isdir(obj): raise Exception( "%s exists and needs to be a dir, but is a %s" % (x.location, obj)) ensure_perms(x, obj) except OSError as oe: if oe.errno != errno.ENOENT: raise try: # we do this form to catch dangling symlinks mkdir(x) except OSError as oe: if oe.errno != errno.EEXIST: raise os.unlink(x.location) mkdir(x) ensure_perms(x) del d # might look odd, but what this does is minimize the try/except cost # to one time, assuming everything behaves, rather then per item. i = iterate(cset.iterdirs(invert=True)) merged_inodes = {} while True: try: for x in i: callback(x) if x.is_reg: key = (x.dev, x.inode) link_target = merged_inodes.get(key) if link_target is not None and \ link_target._can_be_hardlinked(x): if do_link(link_target, x): continue # TODO: should notify that hardlinking failed. merged_inodes.setdefault(key, x) copyfile(x, mkdirs=True) break except CannotOverwrite as cf: if not fs.issym(x): raise # by this time, all directories should've been merged. # thus we can check the target try: if not fs.isdir(gen_obj(pjoin(x.location, x.target))): raise except OSError: raise cf return True
def merge_contents(cset, offset=None, callback=None): """ merge a :class:`pkgcore.fs.contents.contentsSet` instance to the livefs :param cset: :class:`pkgcore.fs.contents.contentsSet` instance :param offset: if not None, offset to prefix all locations with. Think of it as target dir. :param callback: callable to report each entry being merged; given a single arg, the fs object being merged. :raise EnvironmentError: Thrown for permission failures. """ if callback is None: callback = lambda obj:None ensure_perms = get_plugin("fs_ops.ensure_perms") copyfile = get_plugin("fs_ops.copyfile") mkdir = get_plugin("fs_ops.mkdir") if not isinstance(cset, contents.contentsSet): raise TypeError("cset must be a contentsSet, got %r" % (cset,)) if offset is not None: if os.path.exists(offset): if not os.path.isdir(offset): raise TypeError("offset must be a dir, or not exist: %s" % offset) else: mkdir(fs.fsDir(offset, strict=False)) iterate = partial(contents.offset_rewriter, offset.rstrip(os.path.sep)) else: iterate = iter d = list(iterate(cset.iterdirs())) d.sort() for x in d: callback(x) try: # we pass in the stat ourselves, using stat instead of # lstat gen_obj uses internally; this is the equivalent of # "deference that link" obj = gen_obj(x.location, stat=os.stat(x.location)) if not fs.isdir(obj): raise Exception( "%s exists and needs to be a dir, but is a %s" % (x.location, obj)) ensure_perms(x, obj) except OSError as oe: if oe.errno != errno.ENOENT: raise try: # we do this form to catch dangling symlinks mkdir(x) except OSError as oe: if oe.errno != errno.EEXIST: raise os.unlink(x.location) mkdir(x) ensure_perms(x) del d # might look odd, but what this does is minimize the try/except cost # to one time, assuming everything behaves, rather then per item. i = iterate(cset.iterdirs(invert=True)) merged_inodes = {} while True: try: for x in i: callback(x) if x.is_reg: key = (x.dev, x.inode) link_target = merged_inodes.get(key) if link_target is not None and \ link_target._can_be_hardlinked(x): if do_link(link_target, x): continue # TODO: should notify that hardlinking failed. merged_inodes.setdefault(key, x) copyfile(x, mkdirs=True) break except CannotOverwrite as cf: if not fs.issym(x): raise # by this time, all directories should've been merged. # thus we can check the target try: if not fs.isdir(gen_obj(pjoin(x.location, x.target))): raise except OSError: raise cf return True
mode = stat.st_mode d = {"mtime":stat.st_mtime, "mode":S_IMODE(mode), "uid":stat.st_uid, "gid":stat.st_gid} if S_ISREG(mode): d["size"] = stat.st_size d["data"] = local_source(real_location) d["dev"] = stat.st_dev d["inode"] = stat.st_ino if chksum_handlers is not None: d["chf_types"] = chksum_handlers d.update(overrides) return fsFile(path, **d) d.update(overrides) if S_ISDIR(mode): return fsDir(path, **d) elif S_ISLNK(mode): d["target"] = os.readlink(real_location) return fsSymlink(path, **d) elif S_ISFIFO(mode): return fsFifo(path, **d) else: major, minor = get_major_minor(stat) d["minor"] = minor d["major"] = major d["mode"] = mode return fsDev(path, **d) # hmm. this code is roughly 25x slower then find. # make it less slow somehow. the obj instantiation is a bit of a
elif attr != 'chksums': # abuse self as unique singleton. self.assertEqual(getattr(x, attr, self), getattr(y, attr, self)) def test_trigger(self): self.assertContents() self.assertContents( [fs.fsFile("/foon", mode=0644, uid=2, gid=1, strict=False)]) self.assertContents( [fs.fsFile("/foon", mode=0646, uid=1, gid=1, strict=False)]) self.assertContents( [fs.fsFile("/foon", mode=04766, uid=1, gid=2, strict=False)]) self.assertContents([ fs.fsFile("/blarn", mode=02700, uid=2, gid=2, strict=False), fs.fsDir("/dir", mode=0500, uid=2, gid=2, strict=False) ]) self.assertContents([ fs.fsFile("/blarn", mode=02776, uid=2, gid=2, strict=False), fs.fsDir("/dir", mode=02777, uid=1, gid=2, strict=False) ]) self.assertContents([ fs.fsFile("/blarn", mode=06772, uid=2, gid=2, strict=False), fs.fsDir("/dir", mode=04774, uid=1, gid=1, strict=False) ]) class Test_fix_uid_perms(single_attr_change_base, TestCase): kls = triggers.fix_uid_perms attr = 'uid'
def add_missing_directories(self, mode=0775, uid=0, gid=0, mtime=None): """Ensure that a directory node exists for each path; add if missing.""" missing = (x.dirname for x in self) missing = set(x for x in missing if x not in self) if mtime is None: mtime = time.time() # have to go recursive since many directories may be missing. missing_initial = list(missing) for x in missing_initial: target = path.dirname(x) while target not in missing and target not in self: missing.add(target) target = path.dirname(target) missing.discard("/") self.update(fs.fsDir(location=x, mode=mode, uid=uid, gid=gid, mtime=mtime) for x in missing) class OrderedContentsSet(contentsSet): def __init__(self, initial=None, mutable=False, add_missing_directories=False): contentsSet.__init__(self, mutable=True) self._dict = OrderedDict() if initial: self.update(initial) # some sources are a bit stupid, tarballs for example. # add missing directories if requested if add_missing_directories: self.add_missing_directories()
elif attr != 'chksums': # abuse self as unique singleton. self.assertEqual(getattr(x, attr, self), getattr(y, attr, self)) def test_trigger(self): self.assertContents() self.assertContents([fs.fsFile("/foon", mode=0644, uid=2, gid=1, strict=False)]) self.assertContents([fs.fsFile("/foon", mode=0646, uid=1, gid=1, strict=False)]) self.assertContents([fs.fsFile("/foon", mode=04766, uid=1, gid=2, strict=False)]) self.assertContents([fs.fsFile("/blarn", mode=02700, uid=2, gid=2, strict=False), fs.fsDir("/dir", mode=0500, uid=2, gid=2, strict=False)]) self.assertContents([fs.fsFile("/blarn", mode=02776, uid=2, gid=2, strict=False), fs.fsDir("/dir", mode=02777, uid=1, gid=2, strict=False)]) self.assertContents([fs.fsFile("/blarn", mode=06772, uid=2, gid=2, strict=False), fs.fsDir("/dir", mode=04774, uid=1, gid=1, strict=False)]) class Test_fix_uid_perms(single_attr_change_base, TestCase): kls = triggers.fix_uid_perms attr = 'uid' class Test_fix_gid_perms(single_attr_change_base, TestCase):
def merge_contents(cset, offset=None, callback=None): """ merge a :class:`pkgcore.fs.contents.contentsSet` instance to the livefs :param cset: :class:`pkgcore.fs.contents.contentsSet` instance :param offset: if not None, offset to prefix all locations with. Think of it as target dir. :param callback: callable to report each entry being merged; given a single arg, the fs object being merged. :raise EnvironmentError: Thrown for permission failures. """ if callback is None: callback = lambda obj:None ensure_perms = get_plugin("fs_ops.ensure_perms") copyfile = get_plugin("fs_ops.copyfile") mkdir = get_plugin("fs_ops.mkdir") if not isinstance(cset, contents.contentsSet): raise TypeError(f'cset must be a contentsSet, got {cset!r}') if offset is not None: if os.path.exists(offset): if not os.path.isdir(offset): raise TypeError(f'offset must be a dir, or not exist: {offset}') else: mkdir(fs.fsDir(offset, strict=False)) iterate = partial(contents.offset_rewriter, offset.rstrip(os.path.sep)) else: iterate = iter d = list(iterate(cset.iterdirs())) d.sort() for x in d: callback(x) try: # we pass in the stat ourselves, using stat instead of # lstat gen_obj uses internally; this is the equivalent of # "deference that link" obj = gen_obj(x.location, stat=os.stat(x.location)) if not fs.isdir(obj): # according to the spec, dirs can't be merged over files # that aren't dirs or symlinks to dirs raise CannotOverwrite(x.location, obj) ensure_perms(x, obj) except FileNotFoundError: try: # we do this form to catch dangling symlinks mkdir(x) except FileExistsError: os.unlink(x.location) mkdir(x) ensure_perms(x) del d # might look odd, but what this does is minimize the try/except cost # to one time, assuming everything behaves, rather then per item. i = iterate(cset.iterdirs(invert=True)) merged_inodes = {} while True: try: for x in i: callback(x) if x.is_reg: key = (x.dev, x.inode) # This logic could be made smarter- instead of # blindly trying candidates, we could inspect the st_dev # of the final location. This however can be broken by # overlayfs's potentially. Brute force is in use either # way. candidates = merged_inodes.setdefault(key, []) if any(target._can_be_hardlinked(x) and do_link(target, x) for target in candidates): continue candidates.append(x) copyfile(x, mkdirs=True) break except CannotOverwrite as cf: if not fs.issym(x): raise # by this time, all directories should've been merged. # thus we can check the target try: if not fs.isdir(gen_obj(pjoin(x.location, x.target))): raise except OSError: raise cf return True