# Copyright 2004-2011 Brian Harring <*****@*****.**> # Copyright 2006 Marien Zwart <*****@*****.**> # License: BSD/GPL2 """ OS related functionality This module is primarily optimized implementations of various filesystem operations, written for posix specifically. If this is a non-posix system (or extensions were disabled) it falls back to native python implementations that yield no real speed gains. A rough example of the performance benefits, collected from a core2 2.4GHz running python 2.6.5, w/ an EXT4 FS on a 160GB x25-M for the FS related invocations (it's worth noting the IO is pretty fast in this setup- for slow IO like nfs, the speedup for extension vs native for listdir* functionality is a fair bit larger). Rough stats: ======================================================== ========= =============== python -m timeit code snippet native extension time ======================================================== ========= =============== join("/usr/portage", "dev-util", "bsdiff", "ChangeLog") 2.8 usec 0.36 usec normpath("/usr/portage/foon/blah/dar") 5.52 usec 0.15 usec normpath("/usr/portage//foon/blah//dar") 5.66 usec 0.15 usec normpath("/usr/portage/./foon/../blah/") 5.92 usec 0.15 usec listdir_files("/usr/lib64") # 2338 entries, 990 syms 18.6 msec 4.17 msec listdir_files("/usr/lib64", False) # same dir content 16.9 msec 1.48 msec readfile("/etc/passwd") # 1899 bytes 20.4 usec 4.05 usec readfile("tmp-file") # 1MB 300 usec 259 usec list(readlines("/etc/passwd")) # 1899 bytes, 34 lines 37.3 usec 12.8 usec list(readlines("/etc/passwd", False)) # leave whitespace 26.7 usec 12.8 usec ======================================================== ========= ===============
# Copyright 2004-2011 Brian Harring <*****@*****.**> # Copyright 2006 Marien Zwart <*****@*****.**> # License: BSD/GPL2 """ OS related functionality This module is primarily optimized implementations of various filesystem operations, written for posix specifically. If this is a non-posix system (or extensions were disabled) it falls back to native python implementations that yield no real speed gains. A rough example of the performance benefits, collected from a core2 2.4GHz running python 2.6.5, w/ an EXT4 FS on a 160GB x25-M for the FS related invocations (it's worth noting the IO is pretty fast in this setup- for slow IO like nfs, the speedup for extension vs native for listdir* functionality is a fair bit larger). Rough stats: ======================================================== ========= =============== python -m timeit code snippet native extension time ======================================================== ========= =============== join("/usr/portage", "dev-util", "bsdiff", "ChangeLog") 2.8 usec 0.36 usec normpath("/usr/portage/foon/blah/dar") 5.52 usec 0.15 usec normpath("/usr/portage//foon/blah//dar") 5.66 usec 0.15 usec normpath("/usr/portage/./foon/../blah/") 5.92 usec 0.15 usec listdir_files("/usr/lib64") # 2338 entries, 990 syms 18.6 msec 4.17 msec listdir_files("/usr/lib64", False) # same dir content 16.9 msec 1.48 msec readfile("/etc/passwd") # 1899 bytes 20.4 usec 4.05 usec readfile("tmp-file") # 1MB 300 usec 259 usec list(readlines("/etc/passwd")) # 1899 bytes, 34 lines 37.3 usec 12.8 usec list(readlines("/etc/passwd", False)) # leave whitespace 26.7 usec 12.8 usec
def ensure_dirs(path, gid=-1, uid=-1, mode=0o777, minimal=True): """ ensure dirs exist, creating as needed with (optional) gid, uid, and mode. be forewarned- if mode is specified to a mode that blocks the euid from accessing the dir, this code *will* try to create the dir. :param path: directory to ensure exists on disk :param gid: a valid GID to set any created directories to :param uid: a valid UID to set any created directories to :param mode: permissions to set any created directories to :param minimal: boolean controlling whether or not the specified mode must be enforced, or is the minimal permissions necessary. For example, if mode=0755, minimal=True, and a directory exists with mode 0707, this will restore the missing group perms resulting in 757. :return: True if the directory could be created/ensured to have those permissions, False if not. """ try: st = os.stat(path) except OSError: try: um = os.umask(0) # if the dir perms would lack +wx, we have to force it force_temp_perms = ((mode & 0o300) != 0o300) resets = [] apath = normpath(os.path.abspath(path)) sticky_parent = False base = os.path.sep # XXX: wonky way to get filesystem root... prev_path = None prev_mode = None for directory in apath.split(os.path.sep): base = join(base, directory) try: st = os.stat(base) if not stat.S_ISDIR(st.st_mode): return False prev_mode = st.st_mode # if it's a subdir, we need +x at least if apath != base: if (st.st_mode & 0o300) != 0o300: ''' try: os.chmod(base, (st.st_mode | 0o300)) except OSError: return False resets.append((base, st.st_mode)) ''' sticky_parent = (st.st_gid & stat.S_ISGID) except OSError: # nothing exists. if prev_mode & 0300 != 0300: # make sure parent allows for node creation try: os.chmod(prev_path, prev_mode | 0300) resets.append((prev_path, prev_mode)) except OSError: return False try: if force_temp_perms: if not _safe_mkdir(base, 0o700): return False resets.append((base, mode)) else: if not _safe_mkdir(base, mode): return False if base == apath and sticky_parent: resets.append((base, mode)) if gid != -1 or uid != -1: os.chown(base, uid, gid) except OSError: return False prev_path = base try: for base, m in reversed(resets): os.chmod(base, m) if uid != -1 or gid != -1: os.chown(base, uid, gid) except OSError: return False finally: os.umask(um) return True
def ensure_dirs(path, gid=-1, uid=-1, mode=0o777, minimal=True): """ ensure dirs exist, creating as needed with (optional) gid, uid, and mode. be forewarned- if mode is specified to a mode that blocks the euid from accessing the dir, this code *will* try to create the dir. :param path: directory to ensure exists on disk :param gid: a valid GID to set any created directories to :param uid: a valid UID to set any created directories to :param mode: permissions to set any created directories to :param minimal: boolean controlling whether or not the specified mode must be enforced, or is the minimal permissions necessary. For example, if mode=0755, minimal=True, and a directory exists with mode 0707, this will restore the missing group perms resulting in 757. :return: True if the directory could be created/ensured to have those permissions, False if not. """ try: st = os.stat(path) except OSError: base = os.path.sep try: um = os.umask(0) # if the dir perms would lack +wx, we have to force it force_temp_perms = ((mode & 0o300) != 0o300) resets = [] apath = normpath(os.path.abspath(path)) sticky_parent = False for directory in apath.split(os.path.sep): base = join(base, directory) try: st = os.stat(base) if not stat.S_ISDIR(st.st_mode): # one of the path components isn't a dir return False # if it's a subdir, we need +wx at least if apath != base: sticky_parent = (st.st_mode & stat.S_ISGID) except OSError: # nothing exists. try: if force_temp_perms: if not _safe_mkdir(base, 0o700): return False resets.append((base, mode)) else: if not _safe_mkdir(base, mode): return False if base == apath and sticky_parent: resets.append((base, mode)) if gid != -1 or uid != -1: os.chown(base, uid, gid) except OSError: return False try: for base, m in reversed(resets): os.chmod(base, m) if gid != -1 or uid != -1: os.chown(base, uid, gid) except OSError: return False finally: os.umask(um) return True else: if not os.path.isdir(path): # don't change perms for existing paths that aren't dirs return False try: if ((gid != -1 and gid != st.st_gid) or (uid != -1 and uid != st.st_uid)): os.chown(path, uid, gid) if minimal: if mode != (st.st_mode & mode): os.chmod(path, st.st_mode | mode) elif mode != (st.st_mode & 0o7777): os.chmod(path, mode) except OSError: return False return True