def test_copy2_symlinks(self):
tmp_dir = self.mkdtemp()
src = os.path.join(tmp_dir, 'foo')
dst = os.path.join(tmp_dir, 'bar')
src_link = os.path.join(tmp_dir, 'baz')
write_file(src, 'foo')
os.symlink(src, src_link)
if hasattr(os, 'lchmod'):
os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO)
if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'):
os.lchflags(src_link, stat.UF_NODUMP)
src_stat = os.stat(src)
src_link_stat = os.lstat(src_link)
# follow
shutil.copy2(src_link, dst, symlinks=False)
self.assertFalse(os.path.islink(dst))
self.assertEqual(read_file(src), read_file(dst))
os.remove(dst)
# don't follow
shutil.copy2(src_link, dst, symlinks=True)
self.assertTrue(os.path.islink(dst))
self.assertEqual(os.readlink(dst), os.readlink(src_link))
dst_stat = os.lstat(dst)
if hasattr(os, 'lutimes'):
for attr in 'st_atime', 'st_mtime':
# The modification times may be truncated in the new file.
self.assertLessEqual(getattr(src_link_stat, attr),
getattr(dst_stat, attr) + 1)
if hasattr(os, 'lchmod'):
self.assertEqual(src_link_stat.st_mode, dst_stat.st_mode)
self.assertNotEqual(src_stat.st_mode, dst_stat.st_mode)
if hasattr(os, 'lchflags') and hasattr(src_link_stat, 'st_flags'):
self.assertEqual(src_link_stat.st_flags, dst_stat.st_flags)
def create_test_files(self):
"""Create a minimal test case including all supported file types
"""
# File
self.create_regular_file('empty', size=0)
# 2600-01-01 > 2**64 ns
os.utime('input/empty', (19880895600, 19880895600))
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('flagfile', size=1024)
# Directory
self.create_regular_file('dir2/file2', size=1024 * 80)
# File owner
os.chown('input/file1', 100, 200)
# File mode
os.chmod('input/file1', 0o7755)
os.chmod('input/dir2', 0o555)
# Block device
os.mknod('input/bdev', 0o600 | stat.S_IFBLK, os.makedev(10, 20))
# Char device
os.mknod('input/cdev', 0o600 | stat.S_IFCHR, os.makedev(30, 40))
# Hard link
os.link(os.path.join(self.input_path, 'file1'),
os.path.join(self.input_path, 'hardlink'))
# Symlink
os.symlink('somewhere', os.path.join(self.input_path, 'link1'))
if xattr.is_enabled():
xattr.setxattr(os.path.join(self.input_path, 'file1'), 'user.foo', b'bar')
xattr.setxattr(os.path.join(self.input_path, 'link1'), 'user.foo_symlink', b'bar_symlink', follow_symlinks=False)
# FIFO node
os.mkfifo(os.path.join(self.input_path, 'fifo1'))
if has_lchflags:
os.lchflags(os.path.join(self.input_path, 'flagfile'), stat.UF_NODUMP)
def test_copytree_symlinks(self):
tmp_dir = self.mkdtemp()
src_dir = os.path.join(tmp_dir, 'src')
dst_dir = os.path.join(tmp_dir, 'dst')
sub_dir = os.path.join(src_dir, 'sub')
os.mkdir(src_dir)
os.mkdir(sub_dir)
write_file((src_dir, 'file.txt'), 'foo')
src_link = os.path.join(sub_dir, 'link')
dst_link = os.path.join(dst_dir, 'sub/link')
os.symlink(os.path.join(src_dir, 'file.txt'),
src_link)
if hasattr(os, 'lchmod'):
os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO)
if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'):
os.lchflags(src_link, stat.UF_NODUMP)
src_stat = os.lstat(src_link)
shutil.copytree(src_dir, dst_dir, symlinks=True)
self.assertTrue(os.path.islink(os.path.join(dst_dir, 'sub', 'link')))
self.assertEqual(os.readlink(os.path.join(dst_dir, 'sub', 'link')),
os.path.join(src_dir, 'file.txt'))
dst_stat = os.lstat(dst_link)
if hasattr(os, 'lchmod'):
self.assertEqual(dst_stat.st_mode, src_stat.st_mode)
if hasattr(os, 'lchflags'):
self.assertEqual(dst_stat.st_flags, src_stat.st_flags)
def create_test_files(self):
"""Create a minimal test case including all supported file types
"""
# File
self.create_regular_file('empty', size=0)
# 2600-01-01 > 2**64 ns
os.utime('input/empty', (19880895600, 19880895600))
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('flagfile', size=1024)
# Directory
self.create_regular_file('dir2/file2', size=1024 * 80)
# File owner
os.chown('input/file1', 100, 200)
# File mode
os.chmod('input/file1', 0o7755)
os.chmod('input/dir2', 0o555)
# Block device
os.mknod('input/bdev', 0o600 | stat.S_IFBLK, os.makedev(10, 20))
# Char device
os.mknod('input/cdev', 0o600 | stat.S_IFCHR, os.makedev(30, 40))
# Hard link
os.link(os.path.join(self.input_path, 'file1'),
os.path.join(self.input_path, 'hardlink'))
# Symlink
os.symlink('somewhere', os.path.join(self.input_path, 'link1'))
if xattr.is_enabled():
xattr.setxattr(os.path.join(self.input_path, 'file1'), 'user.foo', b'bar')
xattr.setxattr(os.path.join(self.input_path, 'link1'), 'user.foo_symlink', b'bar_symlink', follow_symlinks=False)
# FIFO node
os.mkfifo(os.path.join(self.input_path, 'fifo1'))
if has_lchflags:
os.lchflags(os.path.join(self.input_path, 'flagfile'), stat.UF_NODUMP)
def restore_attrs(self, path, item, symlink=False, fd=None):
xattrs = item.get(b'xattrs', {})
for k, v in xattrs.items():
try:
xattr.setxattr(fd or path, k, v, follow_symlinks=False)
except OSError as e:
if e.errno not in (
errno.ENOTSUP,
errno.EACCES,
):
# only raise if the errno is not on our ignore list:
# ENOTSUP == xattrs not supported here
# EACCES == permission denied to set this specific xattr
# (this may happen related to security.* keys)
raise
uid = gid = None
if not self.numeric_owner:
uid = user2uid(item[b'user'])
gid = group2gid(item[b'group'])
uid = item[b'uid'] if uid is None else uid
gid = item[b'gid'] if gid is None else gid
# This code is a bit of a mess due to os specific differences
try:
if fd:
os.fchown(fd, uid, gid)
else:
os.lchown(path, uid, gid)
except OSError:
pass
if fd:
os.fchmod(fd, item[b'mode'])
elif not symlink:
os.chmod(path, item[b'mode'])
elif has_lchmod: # Not available on Linux
os.lchmod(path, item[b'mode'])
mtime = bigint_to_int(item[b'mtime'])
if b'atime' in item:
atime = bigint_to_int(item[b'atime'])
else:
# old archives only had mtime in item metadata
atime = mtime
if fd:
os.utime(fd, None, ns=(atime, mtime))
else:
os.utime(path, None, ns=(atime, mtime), follow_symlinks=False)
acl_set(path, item, self.numeric_owner)
# Only available on OS X and FreeBSD
if has_lchflags and b'bsdflags' in item:
try:
os.lchflags(path, item[b'bsdflags'])
except OSError:
pass
def copy_metadata(path_src, path_dst, symlink=False):
# print('copystat')
# shutil.copystat(path_src, path_dst)
try:
tmp_flags = 0x0
if symlink:
file_stat = os.lstat(path_src)
dbg_print('lstat: {}'.format(file_stat))
dbg_print('lchown: {} : {} : {}'.format(path_dst, file_stat.st_uid, file_stat.st_gid))
os.lchown(path_dst, file_stat.st_uid, file_stat.st_gid)
dbg_print('lchmod: {}'.format(file_stat.st_mode))
os.lchmod(path_dst, file_stat.st_mode)
else:
file_stat = os.stat(path_src)
dbg_print('stat: {}'.format(file_stat))
dbg_print('chown: {} : {} : {}'.format(path_dst, file_stat.st_uid, file_stat.st_gid))
os.chown(path_dst, file_stat.st_uid, file_stat.st_gid)
dbg_print('copymode')
shutil.copymode(path_src, path_dst)
# Unfortunately, os.utime() of Python 2 does not have the "follow_symlinks" option, so I have no idea to modify atime and mtime of a symlink itself.
# https://stackoverflow.com/questions/48068739/how-can-i-change-atime-and-mtime-of-a-symbolic-link-from-python
dbg_print('utime')
os.utime(path_dst, (file_stat.st_atime, file_stat.st_mtime))
if file_stat.st_flags & stat.SF_NOUNLINK:
tmp_flags |= stat.SF_NOUNLINK
if file_stat.st_flags & 0x80000:
# 0x80000 means SF_RESTRICTED, but Python cannot recognize it.
# https://github.com/pypa/virtualenv/issues/1173
# https://bugs.python.org/issue32347
tmp_flags |= 0x80000
dbg_print('file_stat st_flags ^ tmp_flags: {} | {}'.format(hex(file_stat.st_flags), hex(tmp_flags)))
if symlink:
os.lchflags(path_dst, file_stat.st_flags ^ tmp_flags)
else:
os.chflags(path_dst, file_stat.st_flags ^ tmp_flags)
extattr_src = xattr.xattr(path_src)
extattr_src_items = dict(extattr_src.items())
extattr_dst = xattr.xattr(path_dst)
dbg_print('xattr src: {}'.format(extattr_src.items()))
if 'com.apple.rootless' in extattr_src.keys():
del extattr_src_items['com.apple.rootless']
# dbg_print('xattr dst: {}'.format(extattr_dst.items()))
dbg_print('xattr src: {}'.format(extattr_src_items))
extattr_dst.update(extattr_src_items)
return True
except (IOError, OSError, shutil.Error) as err:
# sys.exit('Error has been occurred in copy_metadata(): {}'.format(err))
return False
def restore_attrs(self, path, item, symlink=False, fd=None):
xattrs = item.get(b'xattrs', {})
for k, v in xattrs.items():
try:
xattr.setxattr(fd or path, k, v, follow_symlinks=False)
except OSError as e:
if e.errno not in (errno.ENOTSUP, errno.EACCES, ):
# only raise if the errno is not on our ignore list:
# ENOTSUP == xattrs not supported here
# EACCES == permission denied to set this specific xattr
# (this may happen related to security.* keys)
raise
uid = gid = None
if not self.numeric_owner:
uid = user2uid(item[b'user'])
gid = group2gid(item[b'group'])
uid = item[b'uid'] if uid is None else uid
gid = item[b'gid'] if gid is None else gid
# This code is a bit of a mess due to os specific differences
try:
if fd:
os.fchown(fd, uid, gid)
else:
os.lchown(path, uid, gid)
except OSError:
pass
if fd:
os.fchmod(fd, item[b'mode'])
elif not symlink:
os.chmod(path, item[b'mode'])
elif has_lchmod: # Not available on Linux
os.lchmod(path, item[b'mode'])
mtime = bigint_to_int(item[b'mtime'])
if b'atime' in item:
atime = bigint_to_int(item[b'atime'])
else:
# old archives only had mtime in item metadata
atime = mtime
if fd and utime_supports_fd: # Python >= 3.3
os.utime(fd, None, ns=(atime, mtime))
elif utime_supports_follow_symlinks: # Python >= 3.3
os.utime(path, None, ns=(atime, mtime), follow_symlinks=False)
elif not symlink:
os.utime(path, (atime / 1e9, mtime / 1e9))
acl_set(path, item, self.numeric_owner)
# Only available on OS X and FreeBSD
if has_lchflags and b'bsdflags' in item:
try:
os.lchflags(path, item[b'bsdflags'])
except OSError:
pass
def create_test_files(self):
"""Create a minimal test case including all supported file types
"""
# File
self.create_regular_file('empty', size=0)
# next code line raises OverflowError on 32bit cpu (raspberry pi 2):
# 2600-01-01 > 2**64 ns
# os.utime('input/empty', (19880895600, 19880895600))
# thus, we better test with something not that far in future:
# 2038-01-19 (1970 + 2^31 - 1 seconds) is the 32bit "deadline":
os.utime('input/empty', (2**31 - 1, 2**31 - 1))
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('flagfile', size=1024)
# Directory
self.create_regular_file('dir2/file2', size=1024 * 80)
# File mode
os.chmod('input/file1', 0o4755)
# Hard link
os.link(os.path.join(self.input_path, 'file1'),
os.path.join(self.input_path, 'hardlink'))
# Symlink
os.symlink('somewhere', os.path.join(self.input_path, 'link1'))
if xattr.is_enabled(self.input_path):
xattr.setxattr(os.path.join(self.input_path, 'file1'), 'user.foo',
b'bar')
# XXX this always fails for me
# ubuntu 14.04, on a TMP dir filesystem with user_xattr, using fakeroot
# same for newer ubuntu and centos.
# if this is supported just on specific platform, platform should be checked first,
# so that the test setup for all tests using it does not fail here always for others.
# xattr.setxattr(os.path.join(self.input_path, 'link1'), 'user.foo_symlink', b'bar_symlink', follow_symlinks=False)
# FIFO node
os.mkfifo(os.path.join(self.input_path, 'fifo1'))
if has_lchflags:
os.lchflags(os.path.join(self.input_path, 'flagfile'),
stat.UF_NODUMP)
try:
# Block device
os.mknod('input/bdev', 0o600 | stat.S_IFBLK, os.makedev(10, 20))
# Char device
os.mknod('input/cdev', 0o600 | stat.S_IFCHR, os.makedev(30, 40))
# File mode
os.chmod(
'input/dir2', 0o555
) # if we take away write perms, we need root to remove contents
# File owner
os.chown('input/file1', 100, 200)
have_root = True # we have (fake)root
except PermissionError:
have_root = False
return have_root
def RemoveFile(path):
try:
os.lchflags(path, 0)
except os.error as e:
pass
try:
os.unlink(path)
except os.error as e:
if e[0] == errno.ENOENT:
return True
else:
print >> sys.stderr, "RemoveFile(%s): errno = %d" % (path, e[0])
return False
return True
def RemoveFile(path):
try:
os.lchflags(path, 0)
except os.error as e:
pass
try:
os.unlink(path)
except os.error as e:
if e[0] == errno.ENOENT:
return True
else:
print >> sys.stderr, "RemoveFile(%s): errno = %d" % (path, e[0])
return False
return True
def RemoveFile(path):
global debug
try:
os.lchflags(path, 0)
except os.error as e:
pass
try:
os.unlink(path)
except os.error as e:
if e[0] == errno.ENOENT:
return True
else:
if debug: log.debug("RemoveFile(%s): errno = %d" % (path, e[0]))
return False
return True
def RemoveFile(path):
global debug
try:
os.lchflags(path, 0)
except os.error as e:
pass
try:
os.unlink(path)
except os.error as e:
if e[0] == errno.ENOENT:
return True
else:
if debug: log.debug("RemoveFile(%s): errno = %d" % (path, e[0]))
return False
return True
def SetPosix(path, meta):
amroot = os.geteuid() == 0
try:
os.lchown(path, meta[TAR_UID_KEY], meta[TAR_GID_KEY])
except os.error as e:
# If we're not root, we can't do the chown
if e[0] != errno.EPERM and amroot:
raise e
os.lchmod(path, meta[TAR_MODE_KEY])
if meta[TAR_FLAGS_KEY] != 0:
try:
os.lchflags(path, meta[TAR_FLAGS_KEY])
except os.error as e:
# If we're not root, we can't do some of this, either
if e[0] != errno.EPERM and amroot:
raise e
def SetPosix(path, meta):
amroot = os.geteuid() == 0
try:
os.lchown(path, meta[TAR_UID_KEY], meta[TAR_GID_KEY])
except os.error as e:
# If we're not root, we can't do the chown
if e[0] != errno.EPERM and amroot:
raise e
os.lchmod(path, meta[TAR_MODE_KEY])
if meta[TAR_FLAGS_KEY] != 0:
try:
os.lchflags(path, meta[TAR_FLAGS_KEY])
except os.error as e:
# If we're not root, we can't do some of this, either
if e[0] != errno.EPERM and amroot:
raise e
def create_test_files(self):
"""Create a minimal test case including all supported file types
"""
# File
self.create_regular_file('empty', size=0)
# next code line raises OverflowError on 32bit cpu (raspberry pi 2):
# 2600-01-01 > 2**64 ns
# os.utime('input/empty', (19880895600, 19880895600))
# thus, we better test with something not that far in future:
# 2038-01-19 (1970 + 2^31 - 1 seconds) is the 32bit "deadline":
os.utime('input/empty', (2**31 - 1, 2**31 - 1))
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('flagfile', size=1024)
# Directory
self.create_regular_file('dir2/file2', size=1024 * 80)
# File mode
os.chmod('input/file1', 0o4755)
# Hard link
os.link(os.path.join(self.input_path, 'file1'),
os.path.join(self.input_path, 'hardlink'))
# Symlink
os.symlink('somewhere', os.path.join(self.input_path, 'link1'))
if xattr.is_enabled(self.input_path):
xattr.setxattr(os.path.join(self.input_path, 'file1'), 'user.foo', b'bar')
# XXX this always fails for me
# ubuntu 14.04, on a TMP dir filesystem with user_xattr, using fakeroot
# same for newer ubuntu and centos.
# if this is supported just on specific platform, platform should be checked first,
# so that the test setup for all tests using it does not fail here always for others.
# xattr.setxattr(os.path.join(self.input_path, 'link1'), 'user.foo_symlink', b'bar_symlink', follow_symlinks=False)
# FIFO node
os.mkfifo(os.path.join(self.input_path, 'fifo1'))
if has_lchflags:
os.lchflags(os.path.join(self.input_path, 'flagfile'), stat.UF_NODUMP)
try:
# Block device
os.mknod('input/bdev', 0o600 | stat.S_IFBLK, os.makedev(10, 20))
# Char device
os.mknod('input/cdev', 0o600 | stat.S_IFCHR, os.makedev(30, 40))
# File mode
os.chmod('input/dir2', 0o555) # if we take away write perms, we need root to remove contents
# File owner
os.chown('input/file1', 100, 200)
have_root = True # we have (fake)root
except PermissionError:
have_root = False
return have_root
def RemoveFile(path):
global debug
try:
os.lchflags(path, 0)
except os.error as e:
pass
try:
os.unlink(path)
except FileNotFoundError:
return True
except OSError as e:
if debug:
log.debug("RemoveFile(%s): errno = %d" % (path, e.errno))
return False
if os.path.exists(path):
raise Exception("After removal, %s still exists" % path)
return True
def RemoveFile(path):
global debug
try:
os.lchflags(path, 0)
except os.error as e:
pass
try:
os.unlink(path)
except os.error as e:
if e[0] == errno.ENOENT:
return True
else:
if debug: log.debug("RemoveFile(%s): errno = %d" % (path, e[0]))
return False
if os.path.exists(path):
raise Exception("After removal, %s still exists" % path)
return True
def RemoveFile(path):
global debug
try:
os.lchflags(path, 0)
except os.error as e:
pass
try:
os.unlink(path)
except (IOError, OSError) as e:
if e.errno == errno.ENOENT:
return True
if debug:
log.debug("RemoveFile(%s): errno = %d" % (path, e.errno))
return False
if os.path.exists(path):
raise Exception("After removal, %s still exists" % path)
return True
def restore_attrs(self, path, item, symlink=False, fd=None):
xattrs = item.get(b'xattrs')
if xattrs:
for k, v in xattrs.items():
try:
xattr.setxattr(fd or path, k, v, follow_symlinks=False)
except OSError as e:
if e.errno != errno.ENOTSUP:
raise
uid = gid = None
if not self.numeric_owner:
uid = user2uid(item[b'user'])
gid = group2gid(item[b'group'])
uid = item[b'uid'] if uid is None else uid
gid = item[b'gid'] if gid is None else gid
# This code is a bit of a mess due to os specific differences
try:
if fd:
os.fchown(fd, uid, gid)
else:
os.lchown(path, uid, gid)
except OSError:
pass
if fd:
os.fchmod(fd, item[b'mode'])
elif not symlink:
os.chmod(path, item[b'mode'])
elif has_lchmod: # Not available on Linux
os.lchmod(path, item[b'mode'])
mtime = bigint_to_int(item[b'mtime'])
if fd and utime_supports_fd: # Python >= 3.3
os.utime(fd, None, ns=(mtime, mtime))
elif utime_supports_follow_symlinks: # Python >= 3.3
os.utime(path, None, ns=(mtime, mtime), follow_symlinks=False)
elif not symlink:
os.utime(path, (mtime / 1e9, mtime / 1e9))
acl_set(path, item, self.numeric_owner)
# Only available on OS X and FreeBSD
if has_lchflags and b'bsdflags' in item:
try:
os.lchflags(path, item[b'bsdflags'])
except OSError:
pass
def restore_attrs(self, path, item, symlink=False, fd=None):
xattrs = item.get(b'xattrs')
if xattrs:
for k, v in xattrs.items():
try:
xattr.setxattr(fd or path, k, v, follow_symlinks=False)
except OSError as e:
if e.errno != errno.ENOTSUP:
raise
uid = gid = None
if not self.numeric_owner:
uid = user2uid(item[b'user'])
gid = group2gid(item[b'group'])
uid = item[b'uid'] if uid is None else uid
gid = item[b'gid'] if gid is None else gid
# This code is a bit of a mess due to os specific differences
try:
if fd:
os.fchown(fd, uid, gid)
else:
os.lchown(path, uid, gid)
except OSError:
pass
if fd:
os.fchmod(fd, item[b'mode'])
elif not symlink:
os.chmod(path, item[b'mode'])
elif has_lchmod: # Not available on Linux
os.lchmod(path, item[b'mode'])
mtime = bigint_to_int(item[b'mtime'])
if fd and utime_supports_fd: # Python >= 3.3
os.utime(fd, None, ns=(mtime, mtime))
elif utime_supports_follow_symlinks: # Python >= 3.3
os.utime(path, None, ns=(mtime, mtime), follow_symlinks=False)
elif not symlink:
os.utime(path, (mtime / 1e9, mtime / 1e9))
acl_set(path, item, self.numeric_owner)
# Only available on OS X and FreeBSD
if has_lchflags and b'bsdflags' in item:
try:
os.lchflags(path, item[b'bsdflags'])
except OSError:
pass
def restore_attrs(self, path, item, symlink=False, fd=None):
xattrs = item.get(b"xattrs")
if xattrs:
for k, v in xattrs.items():
try:
xattr.setxattr(fd or path, k, v, follow_symlinks=False)
except OSError as e:
if e.errno != errno.ENOTSUP:
raise
uid = gid = None
if not self.numeric_owner:
uid = user2uid(item[b"user"])
gid = group2gid(item[b"group"])
uid = item[b"uid"] if uid is None else uid
gid = item[b"gid"] if gid is None else gid
# This code is a bit of a mess due to os specific differences
try:
if fd:
os.fchown(fd, uid, gid)
else:
os.lchown(path, uid, gid)
except OSError:
pass
if fd:
os.fchmod(fd, item[b"mode"])
elif not symlink:
os.chmod(path, item[b"mode"])
elif has_lchmod: # Not available on Linux
os.lchmod(path, item[b"mode"])
if fd and utime_supports_fd: # Python >= 3.3
os.utime(fd, None, ns=(item[b"mtime"], item[b"mtime"]))
elif utime_supports_fd: # Python >= 3.3
os.utime(path, None, ns=(item[b"mtime"], item[b"mtime"]), follow_symlinks=False)
elif not symlink:
os.utime(path, (item[b"mtime"] / 10 ** 9, item[b"mtime"] / 10 ** 9))
acl_set(path, item, self.numeric_owner)
# Only available on OS X and FreeBSD
if has_lchflags and b"bsdflags" in item:
try:
os.lchflags(path, item[b"bsdflags"])
except OSError:
pass
def copystat(src: FileInfo, dst: FileInfo) -> None:
"""Update attribute of dst file with src attributes.
:param src: the source FileInfo object
:param dst: the target FileInfo object
"""
if islink(src): # windows: no cover
mode = stat.S_IMODE(src.stat.st_mode)
if hasattr(os, "lchmod"):
os.lchmod(dst.path, mode)
if hasattr(os, "lchflags") and hasattr(src.stat, "st_flags"):
try:
os.lchflags(dst.path, src.stat.st_flags)
except OSError as why: # defensive code
import errno
if (
not hasattr(errno, "EOPNOTSUPP")
or why.errno != errno.EOPNOTSUPP
):
raise
else:
mode = stat.S_IMODE(src.stat.st_mode)
if hasattr(os, "utime"):
if preserve_timestamps:
os.utime(dst.path, (src.stat.st_atime, src.stat.st_mtime))
else:
os.utime(dst.path, None)
if hasattr(os, "chmod"):
os.chmod(dst.path, mode)
if hasattr(os, "chflags") and hasattr(src.stat, "st_flags"):
try:
os.chflags(dst.path, src.stat.st_flags)
except OSError as why: # defensive code
import errno
if (
not hasattr(errno, "EOPNOTSUPP")
or why.errno != errno.EOPNOTSUPP
):
raise
def RemoveDirectory(path):
st = None
try:
st = os.lstat(path)
except os.error as e:
return False
try:
os.lchflags(path, 0)
except os.error as e:
pass
try:
os.rmdir(path)
except os.error as e:
if st.st_flags:
try:
os.lchflags(path, st.st_flags)
except os.error as e:
pass
return False
return True
def RemoveDirectory(path):
st = None
try:
st = os.lstat(path)
except os.error as e:
return False
try:
os.lchflags(path, 0)
except os.error as e:
pass
try:
os.rmdir(path)
except os.error as e:
if st.st_flags:
try:
os.lchflags(path, st.st_flags)
except os.error as e:
pass
return False
return True
def test_copystat_symlinks(self):
tmp_dir = self.mkdtemp()
src = os.path.join(tmp_dir, 'foo')
dst = os.path.join(tmp_dir, 'bar')
src_link = os.path.join(tmp_dir, 'baz')
dst_link = os.path.join(tmp_dir, 'qux')
write_file(src, 'foo')
src_stat = os.stat(src)
os.utime(src, (src_stat.st_atime,
src_stat.st_mtime - 42.0)) # ensure different mtimes
write_file(dst, 'bar')
self.assertNotEqual(os.stat(src).st_mtime, os.stat(dst).st_mtime)
os.symlink(src, src_link)
os.symlink(dst, dst_link)
if hasattr(os, 'lchmod'):
os.lchmod(src_link, stat.S_IRWXO)
if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'):
os.lchflags(src_link, stat.UF_NODUMP)
src_link_stat = os.lstat(src_link)
# follow
if hasattr(os, 'lchmod'):
shutil.copystat(src_link, dst_link, symlinks=False)
self.assertNotEqual(src_link_stat.st_mode, os.stat(dst).st_mode)
# don't follow
shutil.copystat(src_link, dst_link, symlinks=True)
dst_link_stat = os.lstat(dst_link)
if hasattr(os, 'lutimes'):
for attr in 'st_atime', 'st_mtime':
# The modification times may be truncated in the new file.
self.assertLessEqual(getattr(src_link_stat, attr),
getattr(dst_link_stat, attr) + 1)
if hasattr(os, 'lchmod'):
self.assertEqual(src_link_stat.st_mode, dst_link_stat.st_mode)
if hasattr(os, 'lchflags') and hasattr(src_link_stat, 'st_flags'):
self.assertEqual(src_link_stat.st_flags, dst_link_stat.st_flags)
# tell to follow but dst is not a link
shutil.copystat(src_link, dst, symlinks=True)
self.assertTrue(abs(os.stat(src).st_mtime - os.stat(dst).st_mtime) <
00000.1)
def copystat(src, dst):
"""Update attribute of dst file with src attributes.
:param src: the source FileInfo object
:type src: FileInfo
:param dst: the target FileInfo object
:type dst: FileInfo
"""
if islink(src):
mode = stat.S_IMODE(src.stat.st_mode)
if hasattr(os, 'lchmod'):
os.lchmod(dst.path, mode)
if hasattr(os, 'lchflags') and hasattr(src.stat, 'st_flags'):
try:
os.lchflags(dst.path, src.stat.st_flags)
except OSError as why:
import errno
if (not hasattr(errno, 'EOPNOTSUPP') or
why.errno != errno.EOPNOTSUPP):
raise
else:
mode = stat.S_IMODE(src.stat.st_mode)
if hasattr(os, 'utime'):
if preserve_timestamps:
os.utime(dst.path, (src.stat.st_atime, src.stat.st_mtime))
else:
os.utime(dst.path, None)
if hasattr(os, 'chmod'):
os.chmod(dst.path, mode)
if hasattr(os, 'chflags') and hasattr(src.stat, 'st_flags'):
try:
os.chflags(dst.path, src.stat.st_flags)
except OSError as why:
import errno
if (not hasattr(errno, 'EOPNOTSUPP') or
why.errno != errno.EOPNOTSUPP):
raise
def copystat(src, dst):
"""Update attribute of dst file with src attributes.
:param src: the source FileInfo object
:type src: FileInfo
:param dst: the target FileInfo object
:type dst: FileInfo
"""
if islink(src):
mode = stat.S_IMODE(src.stat.st_mode)
if hasattr(os, 'lchmod'):
os.lchmod(dst.path, mode)
if hasattr(os, 'lchflags') and hasattr(src.stat, 'st_flags'):
try:
os.lchflags(dst.path, src.stat.st_flags)
except OSError as why:
import errno
if (not hasattr(errno, 'EOPNOTSUPP')
or why.errno != errno.EOPNOTSUPP):
raise
else:
mode = stat.S_IMODE(src.stat.st_mode)
if hasattr(os, 'utime'):
if preserve_timestamps:
os.utime(dst.path, (src.stat.st_atime, src.stat.st_mtime))
else:
os.utime(dst.path, None)
if hasattr(os, 'chmod'):
os.chmod(dst.path, mode)
if hasattr(os, 'chflags') and hasattr(src.stat, 'st_flags'):
try:
os.chflags(dst.path, src.stat.st_flags)
except OSError as why:
import errno
if (not hasattr(errno, 'EOPNOTSUPP')
or why.errno != errno.EOPNOTSUPP):
raise
def test_chflags_kwargs():
with Story(['os.lchflags']) as story:
os.lchflags(pth.Path('foo'), stat.UF_NODUMP) == None
with story.replay(strict=True):
pth('foo').chflags(stat.UF_NODUMP, follow_symlinks=True)
def lchflags(self): return os.lchflags(self)
if hasattr(os, 'lchmod'):
def test_lchflags():
with Story(['os.chflags']) as story:
os.lchflags(pth.Path('foo'), stat.UF_NODUMP) == None
with story.replay(strict=True):
pth('foo').lchflags(stat.UF_NODUMP)
def ExtractEntry(tf, entry, root, prefix=None, mFileHash=None):
# This bit of code tries to turn the
# mixture of root, prefix, and pathname into something
# we can both manipulate, and something we can put into
# the database.
# The database should have absolute paths, with no duplicate
# slashes and whatnot. manifest paths come in one of two
# formats, generally: beginning with "./", or beginning with "/"
# So those are the two we look for.
# We also check for root and prefix ending in "/", but the root
# checking is just for prettiness while debugging.
global debug, verbose
fileName = entry.name
if fileName.startswith("./"):
fileName = fileName[2:]
if fileName.startswith("/") or prefix is None:
pass
else:
fileName = "%s%s%s" % (prefix, "" if prefix.endswith("/") or
entry.name.startswith("/") else "/", fileName)
if root:
full_path = "%s%s%s" % (root, "" if root.endswith("/")
or fileName.startswith("/") else "/", fileName)
else:
full_path = "%s%s" % ("" if fileName.startswith("/") else "/",
fileName)
root = ""
# After that, we've got a full_path, and so we get the directory it's in,
# and the name of the file.
dirname = os.path.dirname(full_path)
fname = os.path.basename(full_path)
# Debugging stuff
if debug > 0 or verbose:
log.debug("%s: will be extracted as %s" % (entry.name, full_path))
if debug > 2: log.debug("entry = %s" % (entry))
# Get the metainformation from the TarInfo entry. This is complicated
# because of how flags are done. Note that we don't bother with time
# information.
meta = GetTarMeta(entry)
# Make sure the directory we're creating in exists.
# We don't bother with ownership/mode of the intermediate paths,
# because either it will exist already, or will be part of the
# manifest, in which case posix information will be set. (We
# do use a creation mask of 0755.)
if not os.path.isdir(dirname):
MakeDirs(dirname)
type = None
hash = ""
# Process the entry. We look for a file, directory,
# symlink, or hard link.
if entry.isfile():
fileData = tf.extractfile(entry)
# Is this a problem? Keeping the file in memory?
# Note that we write the file out later, so this allows
# us to not worry about the buffer.
buffer = fileData.read()
hash = hashlib.sha256(buffer).hexdigest()
# PKGNG sets hash to "-" if it's not computed.
if mFileHash != "-":
if hash != mFileHash:
log.error("%s hash does not match manifest" % entry.name)
type = "file"
# First we try to create teh file.
# If that doesn't work, we try to create a
# new file (how would this get cleaned up?),
# and then rename it in place.
# We remove any flags on it -- if there are
# supposed to be any, SetPosix() will get them.
# (We hope.)
try:
os.lchflags(full_path, 0)
except:
pass
newfile = None
try:
f = open(full_path, "w")
except:
newfile = full_path + ".new"
f = open(newfile, "w")
f.write(buffer)
f.close()
if newfile is not None:
try:
os.rename(newfile, full_path)
except:
os.rename(full_path, "%s.old" % full_path)
os.rename(newfile, full_path)
SetPosix(full_path, meta)
elif entry.isdir():
# If the directory already exists, we don't care.
try:
os.makedirs(full_path)
except os.error as e:
if e[0] != errno.EEXIST:
raise e
SetPosix(full_path, meta)
type = "dir"
hash = ""
elif entry.issym():
if mFileHash != "-":
# pkgng now does checksums of symlinks.
# But they remove the leading / for the target,
# so we have to do the same.
if entry.linkname.startswith("/"):
hash = hashlib.sha256(entry.linkname[1:]).hexdigest()
else:
hash = hashlib.sha256(entry.linkname).hexdigest()
if hash != mFileHash:
log.error("%s hash does not match manifest" % entry.name)
hash = ""
else:
# Setting it to '-' will let us know why the entry has no
# checksum.
hash = "-"
# Try to remove the symlink first.
# Then create the new one.
try:
os.unlink(full_path)
except os.error as e:
if e[0] != errno.ENOENT:
log.error("Couldn't unlink %s: %s" % (full_path, e[0]))
raise e
os.symlink(entry.linkname, full_path)
SetPosix(full_path, meta)
type = "slink"
elif entry.islnk():
source_file = root + "/" + entry.linkname
try:
st = os.lstat(source_file)
os.lchflags(source_file, 0)
try:
os.lchflags(full_path, 0)
os.unlink(full_path)
except:
pass
try:
os.link(source_file, full_path)
except os.error as e:
if e[0] == errno.EXDEV:
# Cross-device link, so we'll just copy it
try:
kBufSize = 1024 * 1024
source = open(source_file, "rb")
dest = open(full_path, "wb")
while True:
buffer = source.read(kBufSize)
if buffer:
dest.write(buffer)
else:
break
os.lchmod(full_path, st.st_mode)
except:
log.error("Couldn't copy %s to %s" %
(source_file, full_path))
raise
else:
log.error("Couldn't link %s to %s: %s" %
(source_file, full_path, e[0]))
raise e
if st.st_flags != 0:
os.lchflags(source_file, st.st_flags)
except os.error as e:
log.error("Could not link %s to %s: %s" %
(source_file, full_path, str(e)))
sys.exit(1)
# Except on mac os, hard links are always files.
type = "file"
# Cheating a bit: we'll use the same hash for the hard-link file that's in the pkgng manifest.
hash = mFileHash
if type is not None:
return (fileName, type, hash, meta[TAR_UID_KEY], meta[TAR_GID_KEY],
meta[TAR_FLAGS_KEY], meta[TAR_MODE_KEY])
else:
return None
#!usr/bin/python3
import os, stat
# 打开文件
path = (os.getcwd() + '/lchflags.txt').encode()
fd = os.open(path, os.O_RDWR | os.O_CREAT)
# 关闭文件
os.close(fd)
# 修改标记文件
os.lchflags(path, stat.UF_IMMUTABLE)
# 修改标记成功
print('修改标记成功')
def set_flags(path, bsd_flags, fd=None):
lchflags(path, bsd_flags)
""" 连接对象 """ os.symlink(src,dst) # 创建一个软链接 os.readlink(path) # 返回软链接所指向的文件 os.lstat(path) # 像stat()，但是没有软链接 os.link(src,det) # 创建硬链接，名为参数dst，指向参数src os.lchflags(path,flags) # 设置路径的标记为数字标记,类似chflags()，但是没有软链接 os.lchmod(path,mode) # 修改连接文件权限 os.lchown(path,uid,gid) # 更改文件所有者，类似chown,但是不追踪链接 os.lseek(fd,pos,how) # 设置文件描述符fd当前位置为pos, how方式修改: SEEK_SET 或者 0 设置从文件开始的计算的pos; SEEK_CUR或者 1 则从当前位置计算; os.SEEK_END或者2则从文件尾部开始. 在unix，Windows中有效 """ 文件描述符 """ os.read(fd,n)
def lchflags(self):
return os.lchflags(self)
def chflags(self, flags, follow_symlinks=True):
if follow_symlinks:
os.chflags(self, flags)
else:
os.lchflags(self, flags)
os.dup2(fd, fd2) # 将一个文件描述符 fd 复制到另一个 fd2 os.environ # 获取系统环境变量 os.fchdir(fd) # 通过文件描述符改变当前工作目录 os.fchmod(fd, mode) # 改变一个文件的访问权限，该文件由参数fd指定，参数mode是Unix下的文件访问权限。 os.fchown(fd, uid, gid) # 修改一个文件的所有权，这个函数修改一个文件的用户ID和用户组ID，该文件由文件描述符fd指定。 os.fdatasync(fd) # 强制将文件写入磁盘，该文件由文件描述符fd指定，但是不强制更新文件的状态信息。 os.fdopen(fd[, mode[, bufsize]]) # 通过文件描述符 fd 创建一个文件对象，并返回这个文件对象 os.fpathconf(fd, name) # 返回一个打开的文件的系统配置信息。name为检索的系统配置的值，它也许是一个定义系统值的字符串，这些名字在很多标准中指定（POSIX.1, Unix 95, Unix 98, 和其它）。 os.fstat(fd) # 返回文件描述符fd的状态，像stat()。 os.fstatvfs(fd) # 返回包含文件描述符fd的文件的文件系统的信息，像 statvfs() os.fsync(fd) # 强制将文件描述符为fd的文件写入硬盘。 os.ftruncate(fd, length) # 裁剪文件描述符fd对应的文件, 所以它最大不能超过文件大小。 os.getcwd() # 返回当前工作目录 os.getcwdu() # 返回一个当前工作目录的Unicode对象 os.isatty(fd) # 如果文件描述符fd是打开的，同时与tty(-like)设备相连，则返回true, 否则False。 os.lchflags(path, flags) # 设置路径的标记为数字标记，类似 chflags()，但是没有软链接 os.lchmod(path, mode) # 修改连接文件权限 os.lchown(path, uid, gid) # 更改文件所有者，类似 chown，但是不追踪链接。 os.link(src, dst) # 创建硬链接，名为参数 dst，指向参数 src os.listdir(path) # 返回path指定的文件夹包含的文件或文件夹的名字的列表。 os.lseek(fd, pos, how) # 设置文件描述符 fd当前位置为pos, how方式修改: SEEK_SET 或者 0 设置从文件开始的计算的pos; SEEK_CUR或者 1 则从当前位置计算; os.SEEK_END或者2则从文件尾部开始. 在unix，Windows中有效 os.lstat(path) # 像stat(),但是没有软链接 os.linesep # 当前平台使用的行终止符，win下为"\t\n",Linux下为"\n" os.major(device) # 从原始的设备号中提取设备major号码 (使用stat中的st_dev或者st_rdev field)。 os.makedev(major, minor) # 以major和minor设备号组成一个原始设备号 os.makedirs(path[, mode]) # 递归文件夹创建函数。像mkdir(), 但创建的所有intermediate-level文件夹需要包含子文件夹。 os.minor(device) # 从原始的设备号中提取设备minor号码 (使用stat中的st_dev或者st_rdev field )。 os.mkdir(path[, mode]) # 以数字mode的mode创建一个名为path的文件夹.默认的 mode 是 0777 (八进制)。 os.mkfifo(path[, mode]) # 创建命名管道，mode 为数字，默认为 0666 (八进制) os.mknod(filename[, mode=0600, device]) # 创建一个名为filename文件系统节点（文件，设备特别文件或者命名pipe）。 os.open(file, flags[, mode]) # 打开一个文件，并且设置需要的打开选项，mode参数是可选的
def lchflags(self, flags):
os.lchflags(self, flags)
os.write(fd, "This is test")
# 使用 isatty() 查看文件
ret = os.isatty(fd)
print "返回值: ", ret
# 关闭文件
os.close(fd)
# 24 os.lchflags(path, flags)
# 设置路径的标记为数字标记，类似 chflags()，但是没有软链接
# 打开文件
path = "/var/www/html/foo.txt"
fd = os.open( path, os.O_RDWR|os.O_CREAT )
# 关闭文件
os.close( fd )
# 修改文件标记
ret = os.lchflags(path, os.UF_IMMUTABLE )
print "修改文件标记成功!!"
# 25 os.lchmod(path, mode)
# 修改连接文件权限 只支持在 Unix 下使用。
# 打开文件
path = "/var/www/html/foo.txt"
fd = os.open( path, os.O_RDWR|os.O_CREAT )
# 关闭文件
os.close( fd )
# 修改文件权限
# 设置文件可以通过组执行
os.lchmod( path, stat.S_IXGRP)
# 设置文件可以被其他用户写入
os.lchmod("/tmp/foo.txt", stat.S_IWOTH)
def lchflags(self,flags): return os.lchflags(self.path,flags)
def ExtractEntry(tf, entry, root, prefix = None, mFileHash = None):
# This bit of code tries to turn the
# mixture of root, prefix, and pathname into something
# we can both manipulate, and something we can put into
# the database.
# The database should have absolute paths, with no duplicate
# slashes and whatnot. manifest paths come in one of two
# formats, generally: beginning with "./", or beginning with "/"
# So those are the two we look for.
# We also check for root and prefix ending in "/", but the root
# checking is just for prettiness while debugging.
global debug, verbose
TYPE_DIR = "dir"
TYPE_FILE = "file"
TYPE_SLNK = "slink"
TYPE_OTHER = "unknown"
orig_type = None
new_type = None
fileName = entry.name
if fileName.startswith("./"):
fileName = fileName[2:]
if fileName.startswith("/") or prefix is None:
pass
else:
fileName = "%s%s%s" % (prefix, "" if prefix.endswith("/") or entry.name.startswith("/") else "/", fileName)
if root:
full_path = "%s%s%s" % (root, "" if root.endswith("/") or fileName.startswith("/") else "/", fileName)
else:
full_path = "%s%s" % ("" if fileName.startswith("/") else "/", fileName)
root = ""
try:
import stat
m = os.lstat(full_path).st_mode
if stat.S_ISDIR(m):
orig_type = TYPE_DIR
elif stat.S_ISREG(m):
orig_type = TYPE_FILE
elif stat.S_ISLNK(m):
orig_type = TYPE_SLNK
else:
orig_type = TYPE_OTHER
except:
orig_type = None
# After that, we've got a full_path, and so we get the directory it's in,
# and the name of the file.
dirname = os.path.dirname(full_path)
fname = os.path.basename(full_path)
# Debugging stuff
if debug > 0 or verbose: log.debug("%s: will be extracted as %s" % (entry.name, full_path))
if debug > 2: log.debug("entry = %s" % (entry))
# Get the metainformation from the TarInfo entry. This is complicated
# because of how flags are done. Note that we don't bother with time
# information.
meta = GetTarMeta(entry)
# Make sure the directory we're creating in exists.
# We don't bother with ownership/mode of the intermediate paths,
# because either it will exist already, or will be part of the
# manifest, in which case posix information will be set. (We
# do use a creation mask of 0755.)
if not os.path.isdir(dirname):
MakeDirs(dirname)
type = None
hash = ""
if entry.isfile() or entry.islnk():
new_type = TYPE_FILE
elif entry.isdir():
new_type = TYPE_DIR
elif entry.issym():
new_type = TYPE_SLNK
else:
new_type = TYPE_OTHER
# If the type of the entry changed, but it didn't get removed,
# then bad things could happen. Especially if it changed from
# a symlink to a file or directory.
if orig_type is not None and orig_type != new_type:
log.debug("Original type = %s, new type = %s, path = %s" % (orig_type, new_type, full_path))
log.debug("Removing original entry")
if os.path.islink(full_path) or os.path.isfile(full_path):
RemoveFile(full_path)
elif os.path.isdir(full_path):
import shutil
try:
shutil.rmtree(full_path)
except BaseException as e:
log.error("Couldn't remove old directory %s: %s" % (full_path, str(e)))
raise e
# Process the entry. We look for a file, directory,
# symlink, or hard link.
if entry.isfile():
fileData = tf.extractfile(entry)
# Is this a problem? Keeping the file in memory?
# Note that we write the file out later, so this allows
# us to not worry about the buffer.
buffer = fileData.read()
hash = hashlib.sha256(buffer).hexdigest()
# PKGNG sets hash to "-" if it's not computed.
if mFileHash != "-":
if hash != mFileHash:
log.error("%s hash does not match manifest" % entry.name)
type = "file"
# First we try to create teh file.
# If that doesn't work, we try to create a
# new file (how would this get cleaned up?),
# and then rename it in place.
# We remove any flags on it -- if there are
# supposed to be any, SetPosix() will get them.
# (We hope.)
try:
os.lchflags(full_path, 0)
except:
pass
newfile = None
try:
f = open(full_path, "w")
except:
newfile = full_path + ".new"
f = open(newfile, "w")
f.write(buffer)
f.close()
if newfile is not None:
try:
os.rename(newfile, full_path)
except:
os.rename(full_path, "%s.old" % full_path)
os.rename(newfile, full_path)
SetPosix(full_path, meta)
elif entry.isdir():
# If the directory already exists, we don't care.
try:
os.makedirs(full_path)
except os.error as e:
if e[0] != errno.EEXIST:
raise e
SetPosix(full_path, meta)
type = "dir"
hash = ""
elif entry.issym():
if mFileHash != "-":
# pkgng now does checksums of symlinks.
# But they remove the leading / for the target,
# so we have to do the same.
if entry.linkname.startswith("/"):
hash = hashlib.sha256(entry.linkname[1:]).hexdigest()
else:
hash = hashlib.sha256(entry.linkname).hexdigest()
if hash != mFileHash:
log.error("%s hash does not match manifest" % entry.name)
hash = ""
else:
# Setting it to '-' will let us know why the entry has no
# checksum.
hash = "-"
# Try to remove the symlink first.
# Then create the new one.
try:
os.unlink(full_path)
except os.error as e:
if e[0] == errno.EPERM and os.path.isdir(full_path):
# You can't unlink a directory these days.
import shutil
try:
# This is a truly terrifying thing to do
shutil.rmtree(full_path)
except BaseException as e2:
log.error("Couldn't rmtree %s: %s" % (full_path, str(e2)))
raise e2
elif e[0] != errno.ENOENT:
log.error("Couldn't unlink %s: %s" % (full_path, e[0]))
raise e
os.symlink(entry.linkname, full_path)
SetPosix(full_path, meta)
type = "slink"
elif entry.islnk():
source_file = root + "/" + entry.linkname
try:
st = os.lstat(source_file)
os.lchflags(source_file, 0)
try:
os.lchflags(full_path, 0)
os.unlink(full_path)
except:
pass
try:
os.link(source_file, full_path)
except os.error as e:
if e[0] == errno.EXDEV:
log.debug("Unable to link %s -> %s, trying a copy" % (source_file, full_path))
# Cross-device link, so we'll just copy it
try:
kBufSize = 1024 * 1024
source = open(source_file, "rb")
dest = open(full_path, "wb")
while True:
buffer = source.read(kBufSize)
if buffer:
dest.write(buffer)
else:
break
os.lchmod(full_path, st.st_mode)
except:
log.error("Couldn't copy %s to %s" % (source_file, full_path))
raise
else:
log.error("Couldn't link %s to %s: %s" % (source_file, full_path, e[0]))
raise e
if st.st_flags != 0:
os.lchflags(source_file, st.st_flags)
except os.error as e:
log.error("Could not link %s to %s: %s" % (source_file, full_path, str(e)))
sys.exit(1)
# Except on mac os, hard links are always files.
type = "file"
# Cheating a bit: we'll use the same hash for the hard-link file that's in the pkgng manifest.
hash = mFileHash
if type is not None:
return (fileName,
type,
hash,
meta[TAR_UID_KEY],
meta[TAR_GID_KEY],
meta[TAR_FLAGS_KEY],
meta[TAR_MODE_KEY])
else:
return None
def ExtractEntry(tf, entry, root, prefix=None, mFileHash=None):
# This bit of code tries to turn the
# mixture of root, prefix, and pathname into something
# we can both manipulate, and something we can put into
# the database.
# The database should have absolute paths, with no duplicate
# slashes and whatnot. manifest paths come in one of two
# formats, generally: beginning with "./", or beginning with "/"
# So those are the two we look for.
# We also check for root and prefix ending in "/", but the root
# checking is just for prettiness while debugging.
global debug, verbose
TYPE_DIR = "dir"
TYPE_FILE = "file"
TYPE_SLNK = "slink"
TYPE_OTHER = "unknown"
orig_type = None
new_type = None
fileName = entry.name
if fileName.startswith("./"):
fileName = fileName[2:]
if fileName.startswith("/") or prefix is None:
pass
else:
fileName = "%s%s%s" % (prefix, "" if prefix.endswith("/") or entry.name.startswith("/") else "/", fileName)
if root:
full_path = "%s%s%s" % (root, "" if root.endswith("/") or fileName.startswith("/") else "/", fileName)
else:
full_path = "%s%s" % ("" if fileName.startswith("/") else "/", fileName)
root = ""
try:
m = os.lstat(full_path).st_mode
if stat.S_ISDIR(m):
orig_type = TYPE_DIR
elif stat.S_ISREG(m):
orig_type = TYPE_FILE
elif stat.S_ISLNK(m):
orig_type = TYPE_SLNK
else:
orig_type = TYPE_OTHER
except:
orig_type = None
# After that, we've got a full_path, and so we get the directory it's in,
# and the name of the file.
dirname = os.path.dirname(full_path)
# Debugging stuff
if debug > 0 or verbose:
log.debug("%s: will be extracted as %s" % (entry.name, full_path))
if debug > 2:
log.debug("entry = %s" % (entry))
# Get the metainformation from the TarInfo entry. This is complicated
# because of how flags are done. Note that we don't bother with time
# information.
meta = GetTarMeta(entry)
# Make sure the directory we're creating in exists.
# We don't bother with ownership/mode of the intermediate paths,
# because either it will exist already, or will be part of the
# manifest, in which case posix information will be set. (We
# do use a creation mask of 0755.)
if not os.path.isdir(dirname):
MakeDirs(dirname)
type = None
hash = ""
if entry.isfile() or entry.islnk():
new_type = TYPE_FILE
elif entry.isdir():
new_type = TYPE_DIR
elif entry.issym():
new_type = TYPE_SLNK
else:
new_type = TYPE_OTHER
# If the type of the entry changed, but it didn't get removed,
# then bad things could happen. Especially if it changed from
# a symlink to a file or directory.
if orig_type is not None and orig_type != new_type:
log.debug(
"Original type = %s, new type = %s, path = %s" % (orig_type, new_type, full_path)
)
log.debug("Removing original entry")
if os.path.islink(full_path) or os.path.isfile(full_path):
RemoveFile(full_path)
elif os.path.isdir(full_path):
import shutil
try:
shutil.rmtree(full_path)
except BaseException as e:
log.error("Couldn't remove old directory %s: %s" % (full_path, str(e)))
raise e
# Process the entry. We look for a file, directory,
# symlink, or hard link.
if entry.isfile():
fileData = tf.extractfile(entry)
# Is this a problem? Keeping the file in memory?
# Note that we write the file out later, so this allows
# us to not worry about the buffer.
try:
temp_entry = tempfile.TemporaryFile(dir=os.path.dirname(full_path))
except:
s = "Cannot create temporary file in %s" % os.path.dirname(full_path)
log.error(s)
raise
hash = hashlib.sha256()
while True:
d = fileData.read(1024 * 1024)
if d:
hash.update(d)
temp_entry.write(d)
else:
break
hash = hash.hexdigest()
temp_entry.seek(0)
# PKGNG sets hash to "-" if it's not computed.
if mFileHash != "-":
if hash != mFileHash:
log.error("%s hash does not match manifest" % entry.name)
type = "file"
# First we try to create the file.
# If that doesn't work, we try to create a
# new file (how would this get cleaned up?),
# and then rename it in place.
# We remove any flags on it -- if there are
# supposed to be any, SetPosix() will get them.
# (We hope.)
try:
os.lchflags(full_path, 0)
except:
pass
newfile = None
try:
f = open(full_path, "wb")
except:
newfile = full_path + ".new"
f = open(newfile, "wb")
while True:
d = temp_entry.read(1024 * 1024)
if d:
f.write(d)
else:
break
f.close()
temp_entry.close()
if newfile is not None:
try:
os.rename(newfile, full_path)
except:
os.rename(full_path, "%s.old" % full_path)
os.rename(newfile, full_path)
SetPosix(full_path, meta)
elif entry.isdir():
# If the directory already exists, we don't care.
try:
os.makedirs(full_path)
except (IOError, OSError) as e:
if e.errno != errno.EEXIST:
raise
SetPosix(full_path, meta)
type = "dir"
hash = ""
elif entry.issym():
if mFileHash != "-":
# pkgng now does checksums of symlinks.
# But they remove the leading / for the target,
# so we have to do the same.
if entry.linkname.startswith("/"):
hash = hashlib.sha256(entry.linkname[1:].encode('utf8')).hexdigest()
else:
hash = hashlib.sha256(entry.linkname.encode('utf8')).hexdigest()
if hash != mFileHash:
log.error("%s hash does not match manifest" % entry.name)
hash = ""
else:
# Setting it to '-' will let us know why the entry has no
# checksum.
hash = "-"
# Try to remove the symlink first.
# Then create the new one.
try:
os.unlink(full_path)
except (OSError, IOError) as e:
if e.errno == errno.EPERM and os.path.isdir(full_path):
# You can't unlink a directory these days.
import shutil
try:
# This is a truly terrifying thing to do
shutil.rmtree(full_path)
except BaseException as e2:
log.error("Couldn't rmtree %s: %s" % (full_path, str(e2)))
raise e2
elif e.errno != errno.ENOENT:
log.error("Couldn't unlink %s: %s" % (full_path, e))
raise
os.symlink(entry.linkname, full_path)
SetPosix(full_path, meta)
type = "slink"
elif entry.islnk():
source_file = root + "/" + entry.linkname
try:
st = os.lstat(source_file)
os.lchflags(source_file, 0)
try:
os.lchflags(full_path, 0)
os.unlink(full_path)
except:
pass
try:
os.link(source_file, full_path)
except (IOError, OSError) as e:
if e.errno == errno.EXDEV:
log.debug("Unable to link %s -> %s, trying a copy" % (source_file, full_path))
# Cross-device link, so we'll just copy it
try:
kBufSize = 1024 * 1024
source = open(source_file, "rb")
dest = open(full_path, "wb")
while True:
buffer = source.read(kBufSize)
if buffer:
dest.write(buffer)
else:
break
os.lchmod(full_path, st.st_mode)
except:
log.error("Couldn't copy %s to %s" % (source_file, full_path))
raise
else:
log.error("Couldn't link %s to %s: %s" % (source_file, full_path, e.strerror))
raise e
if st.st_flags != 0:
os.lchflags(source_file, st.st_flags)
except (IOError, OSError) as e:
log.error("Could not link %s to %s: %s" % (source_file, full_path, str(e)))
sys.exit(1)
# Except on mac os, hard links are always files.
type = "file"
# Cheating a bit: we'll use the same hash for the hard-link
# file that's in the pkgng manifest.
hash = mFileHash
if type is not None:
return (fileName,
type,
hash,
meta[TAR_UID_KEY],
meta[TAR_GID_KEY],
meta[TAR_FLAGS_KEY],
meta[TAR_MODE_KEY])
else:
return None
def lchflags(self): return os.lchflags(self)
if hasattr(os, 'lchmod'):
os.chflags("path", stat.UF_NODUMP) # $ getAPathArgument="path"
os.chflags(path="path", flags=stat.UF_NODUMP) # $ getAPathArgument="path"
os.chmod("path", 0o700) # $ getAPathArgument="path"
os.chmod(path="path", mode=0o700) # $ getAPathArgument="path"
os.chown("path", -1, -1) # $ getAPathArgument="path"
os.chown(path="path", uid=-1, gid=-1) # $ getAPathArgument="path"
# unix only
os.chroot("path") # $ getAPathArgument="path"
os.chroot(path="path") # $ getAPathArgument="path"
# unix only
os.lchflags("path", stat.UF_NODUMP) # $ getAPathArgument="path"
os.lchflags(path="path", flags=stat.UF_NODUMP) # $ getAPathArgument="path"
# unix only
os.lchmod("path", 0o700) # $ getAPathArgument="path"
os.lchmod(path="path", mode=0o700) # $ getAPathArgument="path"
# unix only
os.lchown("path", -1, -1) # $ getAPathArgument="path"
os.lchown(path="path", uid=-1, gid=-1) # $ getAPathArgument="path"
os.link("src", "dst") # $ getAPathArgument="src" getAPathArgument="dst"
os.link(src="src",
dst="dst") # $ getAPathArgument="src" getAPathArgument="dst"
os.listdir("path") # $ getAPathArgument="path"
def set_flags(path, bsd_flags, fd=None):
lchflags(path, bsd_flags)
def ExtractEntry(tf, entry, root, prefix = None, mFileHash = None):
# This bit of code tries to turn the
# mixture of root, prefix, and pathname into something
# we can both manipulate, and something we can put into
# the database.
# The database should have absolute paths, with no duplicate
# slashes and whatnot. manifest paths come in one of two
# formats, generally: beginning with "./", or beginning with "/"
# So those are the two we look for.
# We also check for root and prefix ending in "/", but the root
# checking is just for prettiness while debugging.
global debug, verbose
fileName = entry.name
if fileName.startswith("./"):
fileName = fileName[2:]
if fileName.startswith("/") or prefix is None:
pass
else:
fileName = "%s%s%s" % (prefix, "" if prefix.endswith("/") or entry.name.startswith("/") else "/", fileName)
full_path = "%s%s%s" % (root, "" if root.endswith("/") or fileName.startswith("/") else "/", fileName)
# After that, we've got a full_path, and so we get the directory it's in,
# and the name of the file.
dirname = os.path.dirname(full_path)
fname = os.path.basename(full_path)
# Debugging stuff
if debug > 0 or verbose: log.debug("%s: will be extracted as %s" % (entry.name, full_path))
if debug > 2: log.debug("entry = %s" % (entry))
# Get the metainformation from the TarInfo entry. This is complicated
# because of how flags are done. Note that we don't bother with time
# information.
meta = GetTarMeta(entry)
# Make sure the directory we're creating in exists.
# We don't bother with ownership/mode of the intermediate paths,
# because either it will exist already, or will be part of the
# manifest, in which case posix information will be set. (We
# do use a creation mask of 0755.)
if not os.path.isdir(dirname):
MakeDirs(dirname)
type = None
hash = ""
# Process the entry. We look for a file, directory,
# symlink, or hard link.
if entry.isfile():
fileData = tf.extractfile(entry)
# Is this a problem? Keeping the file in memory?
# Note that we write the file out later, so this allows
# us to not worry about the buffer.
buffer = fileData.read()
hash = hashlib.sha256(buffer).hexdigest()
# PKGNG sets hash to "-" if it's not computed.
if mFileHash != "-":
if hash != mFileHash:
log.error("%s hash does not match manifest" % entry.name)
type = "file"
# First we try to create teh file.
# If that doesn't work, we try to create a
# new file (how would this get cleaned up?),
# and then rename it in place.
# We remove any flags on it -- if there are
# supposed to be any, SetPosix() will get them.
# (We hope.)
try:
os.lchflags(full_path, 0)
except:
pass
newfile = None
try:
f = open(full_path, "w")
except:
newfile = full_path + ".new"
f = open(newfile, "w")
f.write(buffer)
f.close()
if newfile is not None:
try:
os.rename(newfile, full_path)
except:
os.rename(full_path, "%s.old" % full_path)
os.rename(newfile, full_path)
SetPosix(full_path, meta)
elif entry.isdir():
# If the directory already exists, we don't care.
try:
os.mkdir(full_path)
except os.error as e:
if e[0] != errno.EEXIST:
raise e
SetPosix(full_path, meta)
type = "dir"
hash = ""
elif entry.issym():
if mFileHash != "-":
# pkgng now does checksums of symlinks.
# But they remove the leading / for the target,
# so we have to do the same.
if entry.linkname.startswith("/"):
hash = hashlib.sha256(entry.linkname[1:]).hexdigest()
else:
hash = hashlib.sha256(entry.linkname).hexdigest()
if hash != mFileHash:
log.error("%s hash does not match manifest" % entry.name)
# Try to remove the symlink first.
# Then create the new one.
try:
os.unlink(full_path)
except os.error as e:
if e[0] != errno.ENOENT:
log.error("Couldn't unlink %s: %s" % (full_path, e[0]))
raise e
os.symlink(entry.linkname, full_path)
SetPosix(full_path, meta)
type = "slink"
hash = ""
elif entry.islnk():
source_file = root + "/" + entry.linkname
try:
st = os.lstat(source_file)
os.lchflags(source_file, 0)
try:
os.lchflags(full_path, 0)
os.unlink(full_path)
except:
pass
os.link(source_file, full_path)
if st.st_flags != 0:
os.lchflags(source_file, st.st_flags)
except os.error as e:
log.error("Could not link %s to %s: %s" % (source_file, full_path, str(e)))
sys.exit(1)
# Except on mac os, hard links are always files.
type = "file"
# Cheating a bit: we'll use the same hash for the hard-link file that's in the pkgng manifest.
hash = mFileHash
if type is not None:
return (fileName,
type,
hash,
meta[TAR_UID_KEY],
meta[TAR_GID_KEY],
meta[TAR_FLAGS_KEY],
meta[TAR_MODE_KEY])
else:
return None
def lchflags(self):
return os.lchflags(self)
assert os.stat(path).st_size == len(test_file_contents)
elif flags & os.O_TRUNC:
assert os.stat(path).st_size == len(test_string)
else:
assert os.stat(path).st_size == (len(test_string) +
len(test_file_contents))
def _has(name):
return pytest.mark.skipif(not hasattr(os, name),
reason="os.{} does not exist".format(name))
@pytest.mark.parametrize("disabled_call", [
_has("chflags")(lambda: os.chflags("/tmp/stuff", stat.SF_IMMUTABLE)),
_has("chroot")(lambda: os.chroot("/tmp/stuff")),
_has("lchflags")(lambda: os.lchflags("/tmp/stuff", stat.SF_IMMUTABLE)),
_has("mkfifo")(lambda: os.mkfifo("/tmp/stuff")),
_has("mkfifo")(lambda: os.mkfifo("/tmp/stuff", 0666)),
_has("mknod")(lambda: os.mknod("/tmp/stuff")),
_has("mknod")(lambda: os.mknod("/tmp/stuff", 0666, 0)),
_has("makedev")(lambda: os.makedev(5, 5)),
_has("pathconf")(lambda: os.pathconf("/tmp/stuff", "PC_FILESIZEBITS")),
_has("removedirs")(lambda: os.removedirs("/tmp/stuff")),
_has("renames")(lambda: os.renames("/tmp/stuff", "/tmp/other-stuff")),
_has("utime")(lambda: os.utime("/tmp/stuff", None)),
])
def test_disabled(disabled_call):
shield.install_hooks()
try:
# With this try-except, other errors will be caught
# and hooks will still be uninstalled