def archive_to_fsobj(src_tar):
psep = os.path.sep
for member in src_tar:
d = {
"uid": member.uid,
"gid": member.gid,
"mtime": member.mtime,
"mode": member.mode
}
location = psep + member.name.strip(psep)
if member.isdir():
if member.name.strip(psep) == ".":
continue
yield fsDir(location, **d)
elif member.isreg():
d["data"] = invokable_data_source.wrap_function(
partial(src_tar.extractfile, member.name),
returns_text=False,
returns_handle=True)
# suppress hardlinks until the rest of pkgcore is updated for it.
d["dev"] = None
d["inode"] = None
yield fsFile(location, **d)
elif member.issym() or member.islnk():
yield fsSymlink(location, member.linkname, **d)
elif member.isfifo():
yield fsFifo(location, **d)
elif member.isdev():
d["major"] = long(member.major)
d["minor"] = long(member.minor)
yield fsDev(location, **d)
else:
raise AssertionError(
"unknown type %r, %r was encounted walking tarmembers" %
(member, member.type))
def archive_to_fsobj(src_tar):
psep = os.path.sep
for member in src_tar:
d = {
"uid":member.uid, "gid":member.gid,
"mtime":member.mtime, "mode":member.mode}
location = psep + member.name.strip(psep)
if member.isdir():
if member.name.strip(psep) == ".":
continue
yield fsDir(location, **d)
elif member.isreg():
d["data"] = invokable_data_source.wrap_function(partial(
src_tar.extractfile, member.name), returns_text=False,
returns_handle=True)
# suppress hardlinks until the rest of pkgcore is updated for it.
d["dev"] = None
d["inode"] = None
yield fsFile(location, **d)
elif member.issym() or member.islnk():
yield fsSymlink(location, member.linkname, **d)
elif member.isfifo():
yield fsFifo(location, **d)
elif member.isdev():
d["major"] = long(member.major)
d["minor"] = long(member.minor)
yield fsDev(location, **d)
else:
raise AssertionError(
"unknown type %r, %r was encounted walking tarmembers" %
(member, member.type))
def gen_obj(path,
stat=None,
chksum_handlers=None,
real_location=None,
stat_func=os.lstat,
**overrides):
"""
given a fs path, and an optional stat, create an appropriate fs obj.
:param stat: stat object to reuse if available
:param real_location: real path to the object if path is the desired
location, rather then existent location.
:raise KeyError: if no obj type matches the stat checks
:return: :obj:`pkgcore.fs.fs.fsBase` derivative
"""
if real_location is None:
real_location = path
if stat is None:
try:
stat = stat_func(real_location)
except EnvironmentError as e:
if stat_func == os.lstat or e.errno != errno.ENOENT:
raise
stat = os.lstat(real_location)
mode = stat.st_mode
d = {
"mtime": stat.st_mtime,
"mode": S_IMODE(mode),
"uid": stat.st_uid,
"gid": stat.st_gid
}
if S_ISREG(mode):
d["size"] = stat.st_size
d["data"] = local_source(real_location)
d["dev"] = stat.st_dev
d["inode"] = stat.st_ino
if chksum_handlers is not None:
d["chf_types"] = chksum_handlers
d.update(overrides)
return fsFile(path, **d)
d.update(overrides)
if S_ISDIR(mode):
return fsDir(path, **d)
elif S_ISLNK(mode):
d["target"] = os.readlink(real_location)
return fsSymlink(path, **d)
elif S_ISFIFO(mode):
return fsFifo(path, **d)
else:
major, minor = get_major_minor(stat)
d["minor"] = minor
d["major"] = major
d["mode"] = mode
return fsDev(path, **d)
def archive_to_fsobj(src_tar):
psep = os.path.sep
dev = _unique_inode()
# inode cache used for supporting hardlinks.
# Since the tarfile specifies a hardlink target by path (rather than internally
# consistent inode numbers), we have to normalize the path lookup into this cache
# via abspath(os.path.join('/', key))...
inodes = {}
for member in src_tar:
d = {
"uid": member.uid,
"gid": member.gid,
"mtime": member.mtime,
"mode": member.mode
}
location = os.path.abspath(os.path.join(psep, member.name.strip(psep)))
if member.isdir():
if member.name.strip(psep) == ".":
continue
yield fsDir(location, **d)
elif member.isreg() or member.islnk():
d["dev"] = dev
if member.islnk():
target = os.path.abspath(os.path.join(psep, member.linkname))
inode = inodes.get(target)
if inode is None:
raise AssertionError(
"Tarfile file %r is a hardlink to %r, but we can't "
"find the resolved hardlink target %r in the archive. "
"This means either a bug in pkgcore, or a malformed "
"tarball." % (member.name, member.linkname, target))
d["inode"] = inode
else:
d["inode"] = inode = _unique_inode()
# Add the new file to the inode cache even if we're currently processing a
# hardlink; tar allows for hardlink chains of x -> y -> z; thus we have
# to ensure 'y' is in the cache alongside it's target z to support 'x'
# later lookup.
inodes[location] = inode
d["data"] = invokable_data_source.wrap_function(
partial(src_tar.extractfile, member.name),
returns_text=False,
returns_handle=True)
yield fsFile(location, **d)
elif member.issym() or member.islnk():
yield fsSymlink(location, member.linkname, **d)
elif member.isfifo():
yield fsFifo(location, **d)
elif member.isdev():
d["major"] = int(member.major)
d["minor"] = int(member.minor)
yield fsDev(location, **d)
else:
raise AssertionError(
"unknown type %r, %r was encounted walking tarmembers" %
(member, member.type))
def test_sym_over_dir(self):
path = pjoin(self.dir, "sym")
fp = pjoin(self.dir, "trg")
os.mkdir(path)
# test sym over a directory.
f = fs.fsSymlink(path, fp, mode=0644, mtime=0, uid=os.getuid(),
gid=os.getgid())
cset = contents.contentsSet([f])
self.assertRaises(ops.FailedCopy, ops.merge_contents, cset)
self.assertTrue(fs.isdir(livefs.gen_obj(path)))
os.mkdir(fp)
ops.merge_contents(cset)
def archive_to_fsobj(src_tar):
psep = os.path.sep
dev = _unique_inode()
# inode cache used for supporting hardlinks.
# Since the tarfile specifies a hardlink target by path (rather than internally
# consistent inode numbers), we have to normalize the path lookup into this cache
# via abspath(os.path.join('/', key))...
inodes = {}
for member in src_tar:
d = {
"uid":member.uid, "gid":member.gid,
"mtime":member.mtime, "mode":member.mode}
location = os.path.abspath(os.path.join(psep, member.name.strip(psep)))
if member.isdir():
if member.name.strip(psep) == ".":
continue
yield fsDir(location, **d)
elif member.isreg() or member.islnk():
d["dev"] = dev
if member.islnk():
target = os.path.abspath(os.path.join(psep, member.linkname))
inode = inodes.get(target)
if inode is None:
raise AssertionError(
"Tarfile file %r is a hardlink to %r, but we can't "
"find the resolved hardlink target %r in the archive. "
"This means either a bug in pkgcore, or a malformed "
"tarball." % (member.name, member.linkname, target))
d["inode"] = inode
else:
d["inode"] = inode = _unique_inode()
# Add the new file to the inode cache even if we're currently processing a
# hardlink; tar allows for hardlink chains of x -> y -> z; thus we have
# to ensure 'y' is in the cache alongside it's target z to support 'x'
# later lookup.
inodes[location] = inode
d["data"] = invokable_data_source.wrap_function(partial(
src_tar.extractfile, member.name), returns_text=False,
returns_handle=True)
yield fsFile(location, **d)
elif member.issym() or member.islnk():
yield fsSymlink(location, member.linkname, **d)
elif member.isfifo():
yield fsFifo(location, **d)
elif member.isdev():
d["major"] = int(member.major)
d["minor"] = int(member.minor)
yield fsDev(location, **d)
else:
raise AssertionError(
"unknown type %r, %r was encounted walking tarmembers" %
(member, member.type))
def test_sym_perms(self):
curgid = os.getgid()
group = [x for x in os.getgroups() if x != curgid]
if not group and os.getuid() != 0:
raise SkipTest("requires root privs for this test, or for this"
" user to belong to more then one group")
group = group[0]
fp = pjoin(self.dir, "sym")
o = fs.fsSymlink(fp, mtime=10321, uid=os.getuid(), gid=group,
mode=0664, target='target')
self.assertTrue(ops.default_copyfile(o))
self.assertEqual(os.lstat(fp).st_gid, group)
self.assertEqual(os.lstat(fp).st_uid, os.getuid())
def gen_obj(path, stat=None, chksum_handlers=None, real_location=None,
stat_func=os.lstat, **overrides):
"""
given a fs path, and an optional stat, create an appropriate fs obj.
:param stat: stat object to reuse if available
:param real_location: real path to the object if path is the desired
location, rather then existent location.
:raise KeyError: if no obj type matches the stat checks
:return: :obj:`pkgcore.fs.fs.fsBase` derivative
"""
if real_location is None:
real_location = path
if stat is None:
try:
stat = stat_func(real_location)
except EnvironmentError as e:
if stat_func == os.lstat or e.errno != errno.ENOENT:
raise
stat = os.lstat(real_location)
mode = stat.st_mode
d = {"mtime":stat.st_mtime, "mode":S_IMODE(mode),
"uid":stat.st_uid, "gid":stat.st_gid}
if S_ISREG(mode):
d["size"] = stat.st_size
d["data"] = local_source(real_location)
d["dev"] = stat.st_dev
d["inode"] = stat.st_ino
if chksum_handlers is not None:
d["chf_types"] = chksum_handlers
d.update(overrides)
return fsFile(path, **d)
d.update(overrides)
if S_ISDIR(mode):
return fsDir(path, **d)
elif S_ISLNK(mode):
d["target"] = os.readlink(real_location)
return fsSymlink(path, **d)
elif S_ISFIFO(mode):
return fsFifo(path, **d)
else:
major, minor = get_major_minor(stat)
d["minor"] = minor
d["major"] = major
d["mode"] = mode
return fsDev(path, **d)
def test_puke_on_dirs(self):
path = pjoin(self.dir, "puke_dir")
self.assertRaises(TypeError,
ops.default_copyfile,
fs.fsDir(path, strict=False))
os.mkdir(path)
fp = pjoin(self.dir, "foon")
open(fp, "w").close()
f = livefs.gen_obj(fp)
self.assertRaises(TypeError,
livefs.gen_obj(fp).change_attributes(location=path))
# test sym over a directory.
f = fs.fsSymlink(path, fp, mode=0644, mtime=0, uid=os.getuid(),
gid=os.getgid())
self.assertRaises(TypeError, ops.default_copyfile, f)
os.unlink(fp)
os.mkdir(fp)
self.assertRaises(ops.CannotOverwrite, ops.default_copyfile, f)
if S_ISREG(mode):
d["size"] = stat.st_size
d["data"] = local_source(real_location)
d["dev"] = stat.st_dev
d["inode"] = stat.st_ino
if chksum_handlers is not None:
d["chf_types"] = chksum_handlers
d.update(overrides)
return fsFile(path, **d)
d.update(overrides)
if S_ISDIR(mode):
return fsDir(path, **d)
elif S_ISLNK(mode):
d["target"] = os.readlink(real_location)
return fsSymlink(path, **d)
elif S_ISFIFO(mode):
return fsFifo(path, **d)
else:
major, minor = get_major_minor(stat)
d["minor"] = minor
d["major"] = major
d["mode"] = mode
return fsDev(path, **d)
# hmm. this code is roughly 25x slower then find.
# make it less slow somehow. the obj instantiation is a bit of a
# killer I'm afraid; without obj, looking at 2.3ms roughly best of 3
# 100 iterations, obj instantiation, 58ms.
# also, os.path.join is rather slow.