# Copyright 2004-2011 Brian Harring <*****@*****.**>
# Copyright 2006 Marien Zwart <*****@*****.**>
# License: BSD/GPL2
"""
OS related functionality
This module is primarily optimized implementations of various filesystem operations,
written for posix specifically. If this is a non-posix system (or extensions were
disabled) it falls back to native python implementations that yield no real speed gains.
A rough example of the performance benefits, collected from a core2 2.4GHz running
python 2.6.5, w/ an EXT4 FS on a 160GB x25-M for the FS related invocations (it's worth
noting the IO is pretty fast in this setup- for slow IO like nfs, the speedup for extension
vs native for listdir* functionality is a fair bit larger).
Rough stats:
======================================================== ========= ===============
python -m timeit code snippet native extension time
======================================================== ========= ===============
join("/usr/portage", "dev-util", "bsdiff", "ChangeLog") 2.8 usec 0.36 usec
normpath("/usr/portage/foon/blah/dar") 5.52 usec 0.15 usec
normpath("/usr/portage//foon/blah//dar") 5.66 usec 0.15 usec
normpath("/usr/portage/./foon/../blah/") 5.92 usec 0.15 usec
listdir_files("/usr/lib64") # 2338 entries, 990 syms 18.6 msec 4.17 msec
listdir_files("/usr/lib64", False) # same dir content 16.9 msec 1.48 msec
readfile("/etc/passwd") # 1899 bytes 20.4 usec 4.05 usec
readfile("tmp-file") # 1MB 300 usec 259 usec
list(readlines("/etc/passwd")) # 1899 bytes, 34 lines 37.3 usec 12.8 usec
list(readlines("/etc/passwd", False)) # leave whitespace 26.7 usec 12.8 usec
======================================================== ========= ===============
# Copyright 2004-2011 Brian Harring <*****@*****.**>
# Copyright 2006 Marien Zwart <*****@*****.**>
# License: BSD/GPL2
"""
OS related functionality
This module is primarily optimized implementations of various filesystem operations,
written for posix specifically. If this is a non-posix system (or extensions were
disabled) it falls back to native python implementations that yield no real speed gains.
A rough example of the performance benefits, collected from a core2 2.4GHz running
python 2.6.5, w/ an EXT4 FS on a 160GB x25-M for the FS related invocations (it's worth
noting the IO is pretty fast in this setup- for slow IO like nfs, the speedup for extension
vs native for listdir* functionality is a fair bit larger).
Rough stats:
======================================================== ========= ===============
python -m timeit code snippet native extension time
======================================================== ========= ===============
join("/usr/portage", "dev-util", "bsdiff", "ChangeLog") 2.8 usec 0.36 usec
normpath("/usr/portage/foon/blah/dar") 5.52 usec 0.15 usec
normpath("/usr/portage//foon/blah//dar") 5.66 usec 0.15 usec
normpath("/usr/portage/./foon/../blah/") 5.92 usec 0.15 usec
listdir_files("/usr/lib64") # 2338 entries, 990 syms 18.6 msec 4.17 msec
listdir_files("/usr/lib64", False) # same dir content 16.9 msec 1.48 msec
readfile("/etc/passwd") # 1899 bytes 20.4 usec 4.05 usec
readfile("tmp-file") # 1MB 300 usec 259 usec
list(readlines("/etc/passwd")) # 1899 bytes, 34 lines 37.3 usec 12.8 usec
list(readlines("/etc/passwd", False)) # leave whitespace 26.7 usec 12.8 usec
def ensure_dirs(path, gid=-1, uid=-1, mode=0o777, minimal=True):
"""
ensure dirs exist, creating as needed with (optional) gid, uid, and mode.
be forewarned- if mode is specified to a mode that blocks the euid
from accessing the dir, this code *will* try to create the dir.
:param path: directory to ensure exists on disk
:param gid: a valid GID to set any created directories to
:param uid: a valid UID to set any created directories to
:param mode: permissions to set any created directories to
:param minimal: boolean controlling whether or not the specified mode
must be enforced, or is the minimal permissions necessary. For example,
if mode=0755, minimal=True, and a directory exists with mode 0707,
this will restore the missing group perms resulting in 757.
:return: True if the directory could be created/ensured to have those
permissions, False if not.
"""
try:
st = os.stat(path)
except OSError:
try:
um = os.umask(0)
# if the dir perms would lack +wx, we have to force it
force_temp_perms = ((mode & 0o300) != 0o300)
resets = []
apath = normpath(os.path.abspath(path))
sticky_parent = False
base = os.path.sep # XXX: wonky way to get filesystem root...
prev_path = None
prev_mode = None
for directory in apath.split(os.path.sep):
base = join(base, directory)
try:
st = os.stat(base)
if not stat.S_ISDIR(st.st_mode):
return False
prev_mode = st.st_mode
# if it's a subdir, we need +x at least
if apath != base:
if (st.st_mode & 0o300) != 0o300:
'''
try:
os.chmod(base, (st.st_mode | 0o300))
except OSError:
return False
resets.append((base, st.st_mode))
'''
sticky_parent = (st.st_gid & stat.S_ISGID)
except OSError:
# nothing exists.
if prev_mode & 0300 != 0300: # make sure parent allows for node creation
try:
os.chmod(prev_path, prev_mode | 0300)
resets.append((prev_path, prev_mode))
except OSError:
return False
try:
if force_temp_perms:
if not _safe_mkdir(base, 0o700):
return False
resets.append((base, mode))
else:
if not _safe_mkdir(base, mode):
return False
if base == apath and sticky_parent:
resets.append((base, mode))
if gid != -1 or uid != -1:
os.chown(base, uid, gid)
except OSError:
return False
prev_path = base
try:
for base, m in reversed(resets):
os.chmod(base, m)
if uid != -1 or gid != -1:
os.chown(base, uid, gid)
except OSError:
return False
finally:
os.umask(um)
return True
# Copyright 2004-2011 Brian Harring <*****@*****.**>
# Copyright 2006 Marien Zwart <*****@*****.**>
# License: BSD/GPL2
"""
OS related functionality
This module is primarily optimized implementations of various filesystem operations,
written for posix specifically. If this is a non-posix system (or extensions were
disabled) it falls back to native python implementations that yield no real speed gains.
A rough example of the performance benefits, collected from a core2 2.4GHz running
python 2.6.5, w/ an EXT4 FS on a 160GB x25-M for the FS related invocations (it's worth
noting the IO is pretty fast in this setup- for slow IO like nfs, the speedup for extension
vs native for listdir* functionality is a fair bit larger).
Rough stats:
======================================================== ========= ===============
python -m timeit code snippet native extension time
======================================================== ========= ===============
join("/usr/portage", "dev-util", "bsdiff", "ChangeLog") 2.8 usec 0.36 usec
normpath("/usr/portage/foon/blah/dar") 5.52 usec 0.15 usec
normpath("/usr/portage//foon/blah//dar") 5.66 usec 0.15 usec
normpath("/usr/portage/./foon/../blah/") 5.92 usec 0.15 usec
listdir_files("/usr/lib64") # 2338 entries, 990 syms 18.6 msec 4.17 msec
listdir_files("/usr/lib64", False) # same dir content 16.9 msec 1.48 msec
readfile("/etc/passwd") # 1899 bytes 20.4 usec 4.05 usec
readfile("tmp-file") # 1MB 300 usec 259 usec
list(readlines("/etc/passwd")) # 1899 bytes, 34 lines 37.3 usec 12.8 usec
list(readlines("/etc/passwd", False)) # leave whitespace 26.7 usec 12.8 usec
def ensure_dirs(path, gid=-1, uid=-1, mode=0o777, minimal=True):
"""
ensure dirs exist, creating as needed with (optional) gid, uid, and mode.
be forewarned- if mode is specified to a mode that blocks the euid
from accessing the dir, this code *will* try to create the dir.
:param path: directory to ensure exists on disk
:param gid: a valid GID to set any created directories to
:param uid: a valid UID to set any created directories to
:param mode: permissions to set any created directories to
:param minimal: boolean controlling whether or not the specified mode
must be enforced, or is the minimal permissions necessary. For example,
if mode=0755, minimal=True, and a directory exists with mode 0707,
this will restore the missing group perms resulting in 757.
:return: True if the directory could be created/ensured to have those
permissions, False if not.
"""
try:
st = os.stat(path)
except OSError:
base = os.path.sep
try:
um = os.umask(0)
# if the dir perms would lack +wx, we have to force it
force_temp_perms = ((mode & 0o300) != 0o300)
resets = []
apath = normpath(os.path.abspath(path))
sticky_parent = False
for directory in apath.split(os.path.sep):
base = join(base, directory)
try:
st = os.stat(base)
if not stat.S_ISDIR(st.st_mode):
# one of the path components isn't a dir
return False
# if it's a subdir, we need +wx at least
if apath != base:
sticky_parent = (st.st_mode & stat.S_ISGID)
except OSError:
# nothing exists.
try:
if force_temp_perms:
if not _safe_mkdir(base, 0o700):
return False
resets.append((base, mode))
else:
if not _safe_mkdir(base, mode):
return False
if base == apath and sticky_parent:
resets.append((base, mode))
if gid != -1 or uid != -1:
os.chown(base, uid, gid)
except OSError:
return False
try:
for base, m in reversed(resets):
os.chmod(base, m)
if gid != -1 or uid != -1:
os.chown(base, uid, gid)
except OSError:
return False
finally:
os.umask(um)
return True
else:
if not os.path.isdir(path):
# don't change perms for existing paths that aren't dirs
return False
try:
if ((gid != -1 and gid != st.st_gid) or
(uid != -1 and uid != st.st_uid)):
os.chown(path, uid, gid)
if minimal:
if mode != (st.st_mode & mode):
os.chmod(path, st.st_mode | mode)
elif mode != (st.st_mode & 0o7777):
os.chmod(path, mode)
except OSError:
return False
return True
def ensure_dirs(path, gid=-1, uid=-1, mode=0o777, minimal=True):
"""
ensure dirs exist, creating as needed with (optional) gid, uid, and mode.
be forewarned- if mode is specified to a mode that blocks the euid
from accessing the dir, this code *will* try to create the dir.
:param path: directory to ensure exists on disk
:param gid: a valid GID to set any created directories to
:param uid: a valid UID to set any created directories to
:param mode: permissions to set any created directories to
:param minimal: boolean controlling whether or not the specified mode
must be enforced, or is the minimal permissions necessary. For example,
if mode=0755, minimal=True, and a directory exists with mode 0707,
this will restore the missing group perms resulting in 757.
:return: True if the directory could be created/ensured to have those
permissions, False if not.
"""
try:
st = os.stat(path)
except OSError:
base = os.path.sep
try:
um = os.umask(0)
# if the dir perms would lack +wx, we have to force it
force_temp_perms = ((mode & 0o300) != 0o300)
resets = []
apath = normpath(os.path.abspath(path))
sticky_parent = False
for directory in apath.split(os.path.sep):
base = join(base, directory)
try:
st = os.stat(base)
if not stat.S_ISDIR(st.st_mode):
# one of the path components isn't a dir
return False
# if it's a subdir, we need +wx at least
if apath != base:
sticky_parent = (st.st_mode & stat.S_ISGID)
except OSError:
# nothing exists.
try:
if force_temp_perms:
if not _safe_mkdir(base, 0o700):
return False
resets.append((base, mode))
else:
if not _safe_mkdir(base, mode):
return False
if base == apath and sticky_parent:
resets.append((base, mode))
if gid != -1 or uid != -1:
os.chown(base, uid, gid)
except OSError:
return False
try:
for base, m in reversed(resets):
os.chmod(base, m)
if gid != -1 or uid != -1:
os.chown(base, uid, gid)
except OSError:
return False
finally:
os.umask(um)
return True
else:
if not os.path.isdir(path):
# don't change perms for existing paths that aren't dirs
return False
try:
if ((gid != -1 and gid != st.st_gid)
or (uid != -1 and uid != st.st_uid)):
os.chown(path, uid, gid)
if minimal:
if mode != (st.st_mode & mode):
os.chmod(path, st.st_mode | mode)
elif mode != (st.st_mode & 0o7777):
os.chmod(path, mode)
except OSError:
return False
return True
# Copyright 2004-2011 Brian Harring <*****@*****.**>
# Copyright 2006 Marien Zwart <*****@*****.**>
# License: BSD/GPL2
"""
OS related functionality
This module is primarily optimized implementations of various filesystem operations,
written for posix specifically. If this is a non-posix system (or extensions were
disabled) it falls back to native python implementations that yield no real speed gains.
A rough example of the performance benefits, collected from a core2 2.4GHz running
python 2.6.5, w/ an EXT4 FS on a 160GB x25-M for the FS related invocations (it's worth
noting the IO is pretty fast in this setup- for slow IO like nfs, the speedup for extension
vs native for listdir* functionality is a fair bit larger).
Rough stats:
======================================================== ========= ===============
python -m timeit code snippet native extension time
======================================================== ========= ===============
join("/usr/portage", "dev-util", "bsdiff", "ChangeLog") 2.8 usec 0.36 usec
normpath("/usr/portage/foon/blah/dar") 5.52 usec 0.15 usec
normpath("/usr/portage//foon/blah//dar") 5.66 usec 0.15 usec
normpath("/usr/portage/./foon/../blah/") 5.92 usec 0.15 usec
listdir_files("/usr/lib64") # 2338 entries, 990 syms 18.6 msec 4.17 msec
listdir_files("/usr/lib64", False) # same dir content 16.9 msec 1.48 msec
readfile("/etc/passwd") # 1899 bytes 20.4 usec 4.05 usec
readfile("tmp-file") # 1MB 300 usec 259 usec
list(readlines("/etc/passwd")) # 1899 bytes, 34 lines 37.3 usec 12.8 usec
list(readlines("/etc/passwd", False)) # leave whitespace 26.7 usec 12.8 usec
======================================================== ========= ===============