def _iter_contents(self): self.clear() for line in self._get_fd(): if not line: continue s = line.split(" ") if s[0] in ("dir", "dev", "fif"): path = ' '.join(s[1:]) if s[0] == 'dir': obj = fs.fsDir(path, strict=False) elif s[0] == 'dev': obj = LookupFsDev(path, strict=False) else: obj = fs.fsFifo(path, strict=False) elif s[0] == "obj": path = ' '.join(s[1:-2]) obj = fs.fsFile( path, chksums={"md5":int(s[-2], 16)}, mtime=int(s[-1]), strict=False) elif s[0] == "sym": try: p = s.index("->") obj = fs.fsLink(' '.join(s[1:p]), ' '.join(s[p+1:-1]), mtime=int(s[-1]), strict=False) except ValueError: # XXX throw a corruption error raise else: raise Exception(f"unknown entry type {line!r}") yield obj
def archive_to_fsobj(src_tar): psep = os.path.sep for member in src_tar: d = { "uid": member.uid, "gid": member.gid, "mtime": member.mtime, "mode": member.mode } location = psep + member.name.strip(psep) if member.isdir(): if member.name.strip(psep) == ".": continue yield fsDir(location, **d) elif member.isreg(): d["data"] = invokable_data_source.wrap_function( partial(src_tar.extractfile, member.name), returns_text=False, returns_handle=True) # suppress hardlinks until the rest of pkgcore is updated for it. d["dev"] = None d["inode"] = None yield fsFile(location, **d) elif member.issym() or member.islnk(): yield fsSymlink(location, member.linkname, **d) elif member.isfifo(): yield fsFifo(location, **d) elif member.isdev(): d["major"] = long(member.major) d["minor"] = long(member.minor) yield fsDev(location, **d) else: raise AssertionError( "unknown type %r, %r was encounted walking tarmembers" % (member, member.type))
def _iter_contents(self): self.clear() for line in self._get_fd(): if not line: continue s = line.split(" ") if s[0] in ("dir", "dev", "fif"): path = ' '.join(s[1:]) if s[0] == 'dir': obj = fs.fsDir(path, strict=False) elif s[0] == 'dev': obj = LookupFsDev(path, strict=False) else: obj = fs.fsFifo(path, strict=False) elif s[0] == "obj": path = ' '.join(s[1:-2]) obj = fs.fsFile( path, chksums={"md5":long(s[-2], 16)}, mtime=long(s[-1]), strict=False) elif s[0] == "sym": try: p = s.index("->") obj = fs.fsLink(' '.join(s[1:p]), ' '.join(s[p+1:-1]), mtime=long(s[-1]), strict=False) except ValueError: # XXX throw a corruption error raise else: raise Exception( "unknown entry type %r" % (line,)) yield obj
def archive_to_fsobj(src_tar): psep = os.path.sep for member in src_tar: d = { "uid":member.uid, "gid":member.gid, "mtime":member.mtime, "mode":member.mode} location = psep + member.name.strip(psep) if member.isdir(): if member.name.strip(psep) == ".": continue yield fsDir(location, **d) elif member.isreg(): d["data"] = invokable_data_source.wrap_function(partial( src_tar.extractfile, member.name), returns_text=False, returns_handle=True) # suppress hardlinks until the rest of pkgcore is updated for it. d["dev"] = None d["inode"] = None yield fsFile(location, **d) elif member.issym() or member.islnk(): yield fsSymlink(location, member.linkname, **d) elif member.isfifo(): yield fsFifo(location, **d) elif member.isdev(): d["major"] = long(member.major) d["minor"] = long(member.minor) yield fsDev(location, **d) else: raise AssertionError( "unknown type %r, %r was encounted walking tarmembers" % (member, member.type))
def gen_obj(path, stat=None, chksum_handlers=None, real_location=None, stat_func=os.lstat, **overrides): """ given a fs path, and an optional stat, create an appropriate fs obj. :param stat: stat object to reuse if available :param real_location: real path to the object if path is the desired location, rather then existent location. :raise KeyError: if no obj type matches the stat checks :return: :obj:`pkgcore.fs.fs.fsBase` derivative """ if real_location is None: real_location = path if stat is None: try: stat = stat_func(real_location) except EnvironmentError as e: if stat_func == os.lstat or e.errno != errno.ENOENT: raise stat = os.lstat(real_location) mode = stat.st_mode d = { "mtime": stat.st_mtime, "mode": S_IMODE(mode), "uid": stat.st_uid, "gid": stat.st_gid } if S_ISREG(mode): d["size"] = stat.st_size d["data"] = local_source(real_location) d["dev"] = stat.st_dev d["inode"] = stat.st_ino if chksum_handlers is not None: d["chf_types"] = chksum_handlers d.update(overrides) return fsFile(path, **d) d.update(overrides) if S_ISDIR(mode): return fsDir(path, **d) elif S_ISLNK(mode): d["target"] = os.readlink(real_location) return fsSymlink(path, **d) elif S_ISFIFO(mode): return fsFifo(path, **d) else: major, minor = get_major_minor(stat) d["minor"] = minor d["major"] = major d["mode"] = mode return fsDev(path, **d)
def archive_to_fsobj(src_tar): psep = os.path.sep dev = _unique_inode() # inode cache used for supporting hardlinks. # Since the tarfile specifies a hardlink target by path (rather than internally # consistent inode numbers), we have to normalize the path lookup into this cache # via abspath(os.path.join('/', key))... inodes = {} for member in src_tar: d = { "uid": member.uid, "gid": member.gid, "mtime": member.mtime, "mode": member.mode } location = os.path.abspath(os.path.join(psep, member.name.strip(psep))) if member.isdir(): if member.name.strip(psep) == ".": continue yield fsDir(location, **d) elif member.isreg() or member.islnk(): d["dev"] = dev if member.islnk(): target = os.path.abspath(os.path.join(psep, member.linkname)) inode = inodes.get(target) if inode is None: raise AssertionError( "Tarfile file %r is a hardlink to %r, but we can't " "find the resolved hardlink target %r in the archive. " "This means either a bug in pkgcore, or a malformed " "tarball." % (member.name, member.linkname, target)) d["inode"] = inode else: d["inode"] = inode = _unique_inode() # Add the new file to the inode cache even if we're currently processing a # hardlink; tar allows for hardlink chains of x -> y -> z; thus we have # to ensure 'y' is in the cache alongside it's target z to support 'x' # later lookup. inodes[location] = inode d["data"] = invokable_data_source.wrap_function( partial(src_tar.extractfile, member.name), returns_text=False, returns_handle=True) yield fsFile(location, **d) elif member.issym() or member.islnk(): yield fsSymlink(location, member.linkname, **d) elif member.isfifo(): yield fsFifo(location, **d) elif member.isdev(): d["major"] = int(member.major) d["minor"] = int(member.minor) yield fsDev(location, **d) else: raise AssertionError( "unknown type %r, %r was encounted walking tarmembers" % (member, member.type))
def archive_to_fsobj(src_tar): psep = os.path.sep dev = _unique_inode() # inode cache used for supporting hardlinks. # Since the tarfile specifies a hardlink target by path (rather than internally # consistent inode numbers), we have to normalize the path lookup into this cache # via abspath(os.path.join('/', key))... inodes = {} for member in src_tar: d = { "uid":member.uid, "gid":member.gid, "mtime":member.mtime, "mode":member.mode} location = os.path.abspath(os.path.join(psep, member.name.strip(psep))) if member.isdir(): if member.name.strip(psep) == ".": continue yield fsDir(location, **d) elif member.isreg() or member.islnk(): d["dev"] = dev if member.islnk(): target = os.path.abspath(os.path.join(psep, member.linkname)) inode = inodes.get(target) if inode is None: raise AssertionError( "Tarfile file %r is a hardlink to %r, but we can't " "find the resolved hardlink target %r in the archive. " "This means either a bug in pkgcore, or a malformed " "tarball." % (member.name, member.linkname, target)) d["inode"] = inode else: d["inode"] = inode = _unique_inode() # Add the new file to the inode cache even if we're currently processing a # hardlink; tar allows for hardlink chains of x -> y -> z; thus we have # to ensure 'y' is in the cache alongside it's target z to support 'x' # later lookup. inodes[location] = inode d["data"] = invokable_data_source.wrap_function(partial( src_tar.extractfile, member.name), returns_text=False, returns_handle=True) yield fsFile(location, **d) elif member.issym() or member.islnk(): yield fsSymlink(location, member.linkname, **d) elif member.isfifo(): yield fsFifo(location, **d) elif member.isdev(): d["major"] = int(member.major) d["minor"] = int(member.minor) yield fsDev(location, **d) else: raise AssertionError( "unknown type %r, %r was encounted walking tarmembers" % (member, member.type))
def gen_obj(path, stat=None, chksum_handlers=None, real_location=None, stat_func=os.lstat, **overrides): """ given a fs path, and an optional stat, create an appropriate fs obj. :param stat: stat object to reuse if available :param real_location: real path to the object if path is the desired location, rather then existent location. :raise KeyError: if no obj type matches the stat checks :return: :obj:`pkgcore.fs.fs.fsBase` derivative """ if real_location is None: real_location = path if stat is None: try: stat = stat_func(real_location) except EnvironmentError as e: if stat_func == os.lstat or e.errno != errno.ENOENT: raise stat = os.lstat(real_location) mode = stat.st_mode d = {"mtime":stat.st_mtime, "mode":S_IMODE(mode), "uid":stat.st_uid, "gid":stat.st_gid} if S_ISREG(mode): d["size"] = stat.st_size d["data"] = local_source(real_location) d["dev"] = stat.st_dev d["inode"] = stat.st_ino if chksum_handlers is not None: d["chf_types"] = chksum_handlers d.update(overrides) return fsFile(path, **d) d.update(overrides) if S_ISDIR(mode): return fsDir(path, **d) elif S_ISLNK(mode): d["target"] = os.readlink(real_location) return fsSymlink(path, **d) elif S_ISFIFO(mode): return fsFifo(path, **d) else: major, minor = get_major_minor(stat) d["minor"] = minor d["major"] = major d["mode"] = mode return fsDev(path, **d)
d["data"] = local_source(real_location) d["dev"] = stat.st_dev d["inode"] = stat.st_ino if chksum_handlers is not None: d["chf_types"] = chksum_handlers d.update(overrides) return fsFile(path, **d) d.update(overrides) if S_ISDIR(mode): return fsDir(path, **d) elif S_ISLNK(mode): d["target"] = os.readlink(real_location) return fsSymlink(path, **d) elif S_ISFIFO(mode): return fsFifo(path, **d) else: major, minor = get_major_minor(stat) d["minor"] = minor d["major"] = major d["mode"] = mode return fsDev(path, **d) # hmm. this code is roughly 25x slower then find. # make it less slow somehow. the obj instantiation is a bit of a # killer I'm afraid; without obj, looking at 2.3ms roughly best of 3 # 100 iterations, obj instantiation, 58ms. # also, os.path.join is rather slow. # in this case, we know it's always pegging one more dir on, so it's # fine doing it this way (specially since we're relying on