def default_copyfile(obj, mkdirs=False):
"""
copy a :class:`pkgcore.fs.fs.fsBase` to its stated location.
:param obj: :class:`pkgcore.fs.fs.fsBase` instance, exempting :class:`fsDir`
:return: true if success, else an exception is thrown
:raise EnvironmentError: permission errors
"""
existent = False
ensure_perms = get_plugin("fs_ops.ensure_perms")
if not fs.isfs_obj(obj):
raise TypeError("obj must be fsBase derivative: %r" % obj)
elif fs.isdir(obj):
raise TypeError("obj must not be a fsDir instance: %r" % obj)
try:
existing = gen_obj(obj.location)
if fs.isdir(existing):
raise CannotOverwrite(obj, existing)
existent = True
except OSError as oe:
# verify the parent dir is there at least
basefp = os.path.dirname(obj.location)
if basefp.strip(os.path.sep) and not os.path.exists(basefp):
if mkdirs:
if not ensure_dirs(basefp, mode=0750, minimal=True):
raise FailedCopy(obj, str(oe))
def default_copyfile(obj, mkdirs=False):
"""
copy a :class:`pkgcore.fs.fs.fsBase` to its stated location.
:param obj: :class:`pkgcore.fs.fs.fsBase` instance, exempting :class:`fsDir`
:return: true if success, else an exception is thrown
:raise EnvironmentError: permission errors
"""
existent = False
ensure_perms = get_plugin("fs_ops.ensure_perms")
if not fs.isfs_obj(obj):
raise TypeError("obj must be fsBase derivative: %r" % obj)
elif fs.isdir(obj):
raise TypeError("obj must not be a fsDir instance: %r" % obj)
try:
existing = gen_obj(obj.location)
if fs.isdir(existing):
raise CannotOverwrite(obj, existing)
existent = True
except OSError as oe:
# verify the parent dir is there at least
basefp = os.path.dirname(obj.location)
if basefp.strip(os.path.sep) and not os.path.exists(basefp):
if mkdirs:
if not ensure_dirs(basefp, mode=0750, minimal=True):
raise FailedCopy(obj, str(oe))
def default_copyfile(obj, mkdirs=False):
"""
copy a :class:`pkgcore.fs.fs.fsBase` to its stated location.
:param obj: :class:`pkgcore.fs.fs.fsBase` instance, exempting :class:`fsDir`
:return: true if success, else an exception is thrown
:raise EnvironmentError: permission errors
"""
existent = False
ensure_perms = get_plugin("fs_ops.ensure_perms")
if not fs.isfs_obj(obj):
raise TypeError(f'obj must be fsBase derivative: {obj!r}')
elif fs.isdir(obj):
raise TypeError(f'obj must not be a fsDir instance: {obj!r}')
try:
existing = gen_obj(obj.location)
if fs.isdir(existing):
raise CannotOverwrite(obj, existing)
existent = True
except OSError as oe:
# verify the parent dir is there at least
basefp = os.path.dirname(obj.location)
if basefp.strip(os.path.sep) and not os.path.exists(basefp):
if mkdirs:
if not ensure_dirs(basefp, mode=0o750, minimal=True):
raise FailedCopy(obj, str(oe))
else:
raise
existent = False
if not existent:
fp = obj.location
else:
fp = existent_fp = obj.location + "#new"
if fs.isreg(obj):
obj.data.transfer_to_path(fp)
elif fs.issym(obj):
os.symlink(obj.target, fp)
elif fs.isfifo(obj):
os.mkfifo(fp)
elif fs.isdev(obj):
dev = os.makedev(obj.major, obj.minor)
os.mknod(fp, obj.mode, dev)
else:
ret = spawn([CP_BINARY, "-Rp", obj.location, fp])
if ret != 0:
raise FailedCopy(obj, f'got {ret} from {CP_BINARY} -Rp')
ensure_perms(obj.change_attributes(location=fp))
if existent:
os.rename(existent_fp, obj.location)
return True
def test_is_funcs(self):
# verify it intercepts the missing attr
self.assertFalse(fs.isdir(object()))
self.assertFalse(fs.isreg(object()))
self.assertFalse(fs.isfifo(object()))
self.assertTrue(fs.isdir(fs.fsDir('/tmp', strict=False)))
self.assertFalse(fs.isreg(fs.fsDir('/tmp', strict=False)))
self.assertTrue(fs.isreg(fs.fsFile('/tmp', strict=False)))
def default_ensure_perms(d1, d2=None):
"""Enforce a fs objects attributes on the livefs.
Attributes enforced are permissions, mtime, uid, gid.
:param d2: if not None, an fs object for what's on the livefs now
:return: True on success, else an exception is thrown
:raise EnvironmentError: if fs object attributes can't be enforced
"""
m, o, g, t = d1.mode, d1.uid, d1.gid, d1.mtime
if o is None:
o = -1
if g is None:
g = -1
if d2 is None:
do_mode, do_chown, do_mtime = True, True, True
else:
do_mode = False
try:
if fs.isdir(d1) and fs.isdir(d2):
# if it's preexisting, keep its perms.
do_mode = False
else:
do_mode = (m is not None and m != d2.mode)
except AttributeError:
# yes. this _is_ stupid. vdb's don't always store all attributes
do_mode = False
do_chown = False
try:
do_chown = (o != d2.uid or g != d2.gid)
except AttributeError:
do_chown = True
try:
do_mtime = (t != d2.mtime)
except AttributeError:
do_mtime = True
if do_chown and (o != -1 or g != -1):
os.lchown(d1.location, o, g)
if not fs.issym(d1):
if do_mode and m is not None:
os.chmod(d1.location, m)
if do_mtime and t is not None:
os.utime(d1.location, (t, t))
return True
def default_ensure_perms(d1, d2=None):
"""Enforce a fs objects attributes on the livefs.
Attributes enforced are permissions, mtime, uid, gid.
:param d2: if not None, an fs object for what's on the livefs now
:return: True on success, else an exception is thrown
:raise EnvironmentError: if fs object attributes can't be enforced
"""
m, o, g, t = d1.mode, d1.uid, d1.gid, d1.mtime
if o is None:
o = -1
if g is None:
g = -1
if d2 is None:
do_mode, do_chown, do_mtime = True, True, True
else:
do_mode = False
try:
if fs.isdir(d1) and fs.isdir(d2):
# if it's preexisting, keep its perms.
do_mode = False
else:
do_mode = (m is not None and m != d2.mode)
except AttributeError:
# yes. this _is_ stupid. vdb's don't always store all attributes
do_mode = False
do_chown = False
try:
do_chown = (o != d2.uid or g != d2.gid)
except AttributeError:
do_chown = True
try:
do_mtime = (t != d2.mtime)
except AttributeError:
do_mtime = True
if do_chown and (o != -1 or g != -1):
os.lchown(d1.location, o, g)
if not fs.issym(d1):
if do_mode and m is not None:
os.chmod(d1.location, m)
if do_mtime and t is not None:
os.utime(d1.location, (t, t))
return True
def test_iterscan(self):
path = os.path.join(self.dir, "iscan")
os.mkdir(path)
files = [
os.path.normpath(os.path.join(path, x))
for x in ["tmp", "blah", "dar"]
]
# cheap version of a touch.
map(lambda x: open(x, "w").close(), files)
dirs = [
os.path.normpath(os.path.join(path, x)) for x in ["a", "b", "c"]
]
map(os.mkdir, dirs)
dirs.append(path)
for obj in livefs.iter_scan(path):
self.assertInstance(obj, fs.fsBase)
if fs.isreg(obj):
self.assertTrue(obj.location in files)
elif fs.isdir(obj):
self.assertTrue(obj.location in dirs)
else:
raise Exception(
"unknown object popped up in testing dir, '%s'" % obj)
self.check_attrs(obj, obj.location)
# do offset verification now.
offset = os.path.join(self.dir, "iscan")
for obj in livefs.iter_scan(path, offset=offset):
self.check_attrs(obj, obj.location, offset=offset)
seen = []
for obj in livefs.iter_scan(files[0]):
self.check_attrs(obj, obj.location)
seen.append(obj.location)
self.assertEqual((files[0], ), tuple(sorted(seen)))
def test_iterscan(self):
path = os.path.join(self.dir, "iscan")
os.mkdir(path)
files = [os.path.normpath(os.path.join(path, x)) for x in [
"tmp", "blah", "dar"]]
# cheap version of a touch.
map(lambda x:open(x, "w").close(), files)
dirs = [os.path.normpath(os.path.join(path, x)) for x in [
"a", "b", "c"]]
map(os.mkdir, dirs)
dirs.append(path)
for obj in livefs.iter_scan(path):
self.assertInstance(obj, fs.fsBase)
if fs.isreg(obj):
self.assertTrue(obj.location in files)
elif fs.isdir(obj):
self.assertTrue(obj.location in dirs)
else:
raise Exception(
"unknown object popped up in testing dir, '%s'" % obj)
self.check_attrs(obj, obj.location)
# do offset verification now.
offset = os.path.join(self.dir, "iscan")
for obj in livefs.iter_scan(path, offset=offset):
self.check_attrs(obj, obj.location, offset=offset)
seen = []
for obj in livefs.iter_scan(files[0]):
self.check_attrs(obj, obj.location)
seen.append(obj.location)
self.assertEqual((files[0],), tuple(sorted(seen)))
def merge_contents(cset, offset=None, callback=None):
"""
merge a :class:`pkgcore.fs.contents.contentsSet` instance to the livefs
:param cset: :class:`pkgcore.fs.contents.contentsSet` instance
:param offset: if not None, offset to prefix all locations with.
Think of it as target dir.
:param callback: callable to report each entry being merged; given a single arg,
the fs object being merged.
:raise EnvironmentError: Thrown for permission failures.
"""
if callback is None:
callback = lambda obj:None
ensure_perms = get_plugin("fs_ops.ensure_perms")
copyfile = get_plugin("fs_ops.copyfile")
mkdir = get_plugin("fs_ops.mkdir")
if not isinstance(cset, contents.contentsSet):
raise TypeError("cset must be a contentsSet, got %r" % (cset,))
if offset is not None:
if os.path.exists(offset):
if not os.path.isdir(offset):
raise TypeError("offset must be a dir, or not exist: %s" % offset)
else:
mkdir(fs.fsDir(offset, strict=False))
iterate = partial(contents.offset_rewriter, offset.rstrip(os.path.sep))
else:
iterate = iter
d = list(iterate(cset.iterdirs()))
d.sort()
for x in d:
callback(x)
try:
# we pass in the stat ourselves, using stat instead of
# lstat gen_obj uses internally; this is the equivalent of
# "deference that link"
obj = gen_obj(x.location, stat=os.stat(x.location))
if not fs.isdir(obj):
raise Exception(
"%s exists and needs to be a dir, but is a %s" %
(x.location, obj))
ensure_perms(x, obj)
except OSError, oe:
if oe.errno != errno.ENOENT:
raise
try:
# we do this form to catch dangling symlinks
mkdir(x)
except OSError, oe:
if oe.errno != errno.EEXIST:
raise
os.unlink(x.location)
mkdir(x)
ensure_perms(x)
def _scan_mtimes(locations, stat_func):
for x in locations:
try:
st = stat_func(x)
except FileNotFoundError:
continue
obj = gen_obj(x, stat=st)
if fs.isdir(obj):
yield obj
def test_sym_over_dir(self):
path = pjoin(self.dir, "sym")
fp = pjoin(self.dir, "trg")
os.mkdir(path)
# test sym over a directory.
f = fs.fsSymlink(path, fp, mode=0644, mtime=0, uid=os.getuid(),
gid=os.getgid())
cset = contents.contentsSet([f])
self.assertRaises(ops.FailedCopy, ops.merge_contents, cset)
self.assertTrue(fs.isdir(livefs.gen_obj(path)))
os.mkdir(fp)
ops.merge_contents(cset)
def test_read_ld_so_conf(self):
# test the defaults first. should create etc and the file.
self.assertPaths(self.trigger.read_ld_so_conf(self.dir),
[pjoin(self.dir, x) for x in self.trigger.default_ld_path])
o = gen_obj(pjoin(self.dir, 'etc'))
self.assertEqual(o.mode, 0755)
self.assertTrue(fs.isdir(o))
self.assertTrue(os.path.exists(pjoin(self.dir, 'etc/ld.so.conf')))
# test normal functioning.
with open(pjoin(self.dir, 'etc/ld.so.conf'), 'w') as f:
f.write("\n".join(["/foon", "dar", "blarnsball", "#comment"]))
self.assertPaths(self.trigger.read_ld_so_conf(self.dir),
[pjoin(self.dir, x) for x in ["foon", "dar", "blarnsball"]])
def test_read_ld_so_conf(self):
# test the defaults first. should create etc and the file.
self.assertPaths(self.trigger.read_ld_so_conf(self.dir),
[pjoin(self.dir, x) for x in self.trigger.default_ld_path])
o = gen_obj(pjoin(self.dir, 'etc'))
self.assertEqual(o.mode, 0755)
self.assertTrue(fs.isdir(o))
self.assertTrue(os.path.exists(pjoin(self.dir, 'etc/ld.so.conf')))
# test normal functioning.
open(pjoin(self.dir, 'etc/ld.so.conf'), 'w').write("\n".join(
["/foon", "dar", "blarnsball", "#comment"]))
self.assertPaths(self.trigger.read_ld_so_conf(self.dir),
[pjoin(self.dir, x) for x in ["foon", "dar", "blarnsball"]])
def merge_contents(cset, offset=None, callback=None):
"""
merge a :class:`pkgcore.fs.contents.contentsSet` instance to the livefs
:param cset: :class:`pkgcore.fs.contents.contentsSet` instance
:param offset: if not None, offset to prefix all locations with.
Think of it as target dir.
:param callback: callable to report each entry being merged; given a single arg,
the fs object being merged.
:raise EnvironmentError: Thrown for permission failures.
"""
if callback is None:
callback = lambda obj: None
ensure_perms = get_plugin("fs_ops.ensure_perms")
copyfile = get_plugin("fs_ops.copyfile")
mkdir = get_plugin("fs_ops.mkdir")
if not isinstance(cset, contents.contentsSet):
raise TypeError("cset must be a contentsSet, got %r" % (cset, ))
if offset is not None:
if os.path.exists(offset):
if not os.path.isdir(offset):
raise TypeError("offset must be a dir, or not exist: %s" %
offset)
else:
mkdir(fs.fsDir(offset, strict=False))
iterate = partial(contents.offset_rewriter, offset.rstrip(os.path.sep))
else:
iterate = iter
d = list(iterate(cset.iterdirs()))
d.sort()
for x in d:
callback(x)
try:
# we pass in the stat ourselves, using stat instead of
# lstat gen_obj uses internally; this is the equivalent of
# "deference that link"
obj = gen_obj(x.location, stat=os.stat(x.location))
if not fs.isdir(obj):
raise Exception(
"%s exists and needs to be a dir, but is a %s" %
(x.location, obj))
ensure_perms(x, obj)
except OSError as oe:
if oe.errno != errno.ENOENT:
raise
try:
# we do this form to catch dangling symlinks
mkdir(x)
except OSError as oe:
if oe.errno != errno.EEXIST:
raise
os.unlink(x.location)
mkdir(x)
ensure_perms(x)
del d
# might look odd, but what this does is minimize the try/except cost
# to one time, assuming everything behaves, rather then per item.
i = iterate(cset.iterdirs(invert=True))
merged_inodes = {}
while True:
try:
for x in i:
callback(x)
if x.is_reg:
key = (x.dev, x.inode)
link_target = merged_inodes.get(key)
if link_target is not None and \
link_target._can_be_hardlinked(x):
if do_link(link_target, x):
continue
# TODO: should notify that hardlinking failed.
merged_inodes.setdefault(key, x)
copyfile(x, mkdirs=True)
break
except CannotOverwrite as cf:
if not fs.issym(x):
raise
# by this time, all directories should've been merged.
# thus we can check the target
try:
if not fs.isdir(gen_obj(pjoin(x.location, x.target))):
raise
except OSError:
raise cf
return True
def merge_contents(cset, offset=None, callback=None):
"""
merge a :class:`pkgcore.fs.contents.contentsSet` instance to the livefs
:param cset: :class:`pkgcore.fs.contents.contentsSet` instance
:param offset: if not None, offset to prefix all locations with.
Think of it as target dir.
:param callback: callable to report each entry being merged; given a single arg,
the fs object being merged.
:raise EnvironmentError: Thrown for permission failures.
"""
if callback is None:
callback = lambda obj:None
ensure_perms = get_plugin("fs_ops.ensure_perms")
copyfile = get_plugin("fs_ops.copyfile")
mkdir = get_plugin("fs_ops.mkdir")
if not isinstance(cset, contents.contentsSet):
raise TypeError(f'cset must be a contentsSet, got {cset!r}')
if offset is not None:
if os.path.exists(offset):
if not os.path.isdir(offset):
raise TypeError(f'offset must be a dir, or not exist: {offset}')
else:
mkdir(fs.fsDir(offset, strict=False))
iterate = partial(contents.offset_rewriter, offset.rstrip(os.path.sep))
else:
iterate = iter
d = list(iterate(cset.iterdirs()))
d.sort()
for x in d:
callback(x)
try:
# we pass in the stat ourselves, using stat instead of
# lstat gen_obj uses internally; this is the equivalent of
# "deference that link"
obj = gen_obj(x.location, stat=os.stat(x.location))
if not fs.isdir(obj):
# according to the spec, dirs can't be merged over files
# that aren't dirs or symlinks to dirs
raise CannotOverwrite(x.location, obj)
ensure_perms(x, obj)
except FileNotFoundError:
try:
# we do this form to catch dangling symlinks
mkdir(x)
except FileExistsError:
os.unlink(x.location)
mkdir(x)
ensure_perms(x)
del d
# might look odd, but what this does is minimize the try/except cost
# to one time, assuming everything behaves, rather then per item.
i = iterate(cset.iterdirs(invert=True))
merged_inodes = {}
while True:
try:
for x in i:
callback(x)
if x.is_reg:
key = (x.dev, x.inode)
# This logic could be made smarter- instead of
# blindly trying candidates, we could inspect the st_dev
# of the final location. This however can be broken by
# overlayfs's potentially. Brute force is in use either
# way.
candidates = merged_inodes.setdefault(key, [])
if any(target._can_be_hardlinked(x) and do_link(target, x)
for target in candidates):
continue
candidates.append(x)
copyfile(x, mkdirs=True)
break
except CannotOverwrite as cf:
if not fs.issym(x):
raise
# by this time, all directories should've been merged.
# thus we can check the target
try:
if not fs.isdir(gen_obj(pjoin(x.location, x.target))):
raise
except OSError:
raise cf
return True
def test_gen_obj_dir(self):
o = livefs.gen_obj(self.dir)
self.assertTrue(fs.isdir(o))
self.check_attrs(o, self.dir)
def merge_contents(cset, offset=None, callback=None):
"""
merge a :class:`pkgcore.fs.contents.contentsSet` instance to the livefs
:param cset: :class:`pkgcore.fs.contents.contentsSet` instance
:param offset: if not None, offset to prefix all locations with.
Think of it as target dir.
:param callback: callable to report each entry being merged; given a single arg,
the fs object being merged.
:raise EnvironmentError: Thrown for permission failures.
"""
if callback is None:
callback = lambda obj:None
ensure_perms = get_plugin("fs_ops.ensure_perms")
copyfile = get_plugin("fs_ops.copyfile")
mkdir = get_plugin("fs_ops.mkdir")
if not isinstance(cset, contents.contentsSet):
raise TypeError("cset must be a contentsSet, got %r" % (cset,))
if offset is not None:
if os.path.exists(offset):
if not os.path.isdir(offset):
raise TypeError("offset must be a dir, or not exist: %s" % offset)
else:
mkdir(fs.fsDir(offset, strict=False))
iterate = partial(contents.offset_rewriter, offset.rstrip(os.path.sep))
else:
iterate = iter
d = list(iterate(cset.iterdirs()))
d.sort()
for x in d:
callback(x)
try:
# we pass in the stat ourselves, using stat instead of
# lstat gen_obj uses internally; this is the equivalent of
# "deference that link"
obj = gen_obj(x.location, stat=os.stat(x.location))
if not fs.isdir(obj):
raise Exception(
"%s exists and needs to be a dir, but is a %s" %
(x.location, obj))
ensure_perms(x, obj)
except OSError as oe:
if oe.errno != errno.ENOENT:
raise
try:
# we do this form to catch dangling symlinks
mkdir(x)
except OSError as oe:
if oe.errno != errno.EEXIST:
raise
os.unlink(x.location)
mkdir(x)
ensure_perms(x)
del d
# might look odd, but what this does is minimize the try/except cost
# to one time, assuming everything behaves, rather then per item.
i = iterate(cset.iterdirs(invert=True))
merged_inodes = {}
while True:
try:
for x in i:
callback(x)
if x.is_reg:
key = (x.dev, x.inode)
link_target = merged_inodes.get(key)
if link_target is not None and \
link_target._can_be_hardlinked(x):
if do_link(link_target, x):
continue
# TODO: should notify that hardlinking failed.
merged_inodes.setdefault(key, x)
copyfile(x, mkdirs=True)
break
except CannotOverwrite as cf:
if not fs.issym(x):
raise
# by this time, all directories should've been merged.
# thus we can check the target
try:
if not fs.isdir(gen_obj(pjoin(x.location, x.target))):
raise
except OSError:
raise cf
return True
def test_gen_obj_dir(self):
o = livefs.gen_obj(self.dir)
self.assertTrue(fs.isdir(o))
self.check_attrs(o, self.dir)
link_target._can_be_hardlinked(x):
if do_link(link_target, x):
continue
# TODO: should notify that hardlinking failed.
merged_inodes.setdefault(key, x)
copyfile(x, mkdirs=True)
break
except CannotOverwrite, cf:
if not fs.issym(x):
raise
# by this time, all directories should've been merged.
# thus we can check the target
try:
if not fs.isdir(gen_obj(pjoin(x.location, x.target))):
raise
except OSError:
raise cf
return True
def unmerge_contents(cset, offset=None, callback=None):
"""
unmerge a :obj:`pkgcore.fs.contents.contentsSet` instance to the livefs
:param cset: :obj:`pkgcore.fs.contents.contentsSet` instance
:param offset: if not None, offset to prefix all locations with.
Think of it as target dir.
:param callback: callable to report each entry being unmerged
