def ensure_dirs(dir_path, *args, **kwargs):
"""Create a directory and call apply_permissions.
Returns True if a directory is created or the permissions needed to be
modified, and False otherwise."""
created_dir = False
try:
os.makedirs(dir_path)
created_dir = True
except OSError as oe:
func_call = "makedirs('%s')" % dir_path
if oe.errno in (errno.EEXIST, errno.EISDIR):
pass
else:
if os.path.isdir(dir_path):
# NOTE: DragonFly raises EPERM for makedir('/')
# and that is supposed to be ignored here.
pass
elif oe.errno == errno.EPERM:
raise OperationNotPermitted(func_call)
elif oe.errno == errno.EACCES:
raise PermissionDenied(func_call)
elif oe.errno == errno.EROFS:
raise ReadOnlyFileSystem(func_call)
else:
raise
perms_modified = apply_permissions(dir_path, *args, **kwargs)
return created_dir or perms_modified
def write_atomic(file_path, content, **kwargs):
f = None
try:
f = atomic_ofstream(file_path, **kwargs)
f.write(content)
f.close()
except (IOError, OSError) as e:
if f:
f.abort()
func_call = "write_atomic('%s')" % file_path
if e.errno == errno.EPERM:
raise OperationNotPermitted(func_call)
elif e.errno == errno.EACCES:
raise PermissionDenied(func_call)
elif e.errno == errno.EROFS:
raise ReadOnlyFileSystem(func_call)
elif e.errno == errno.ENOENT:
raise FileNotFound(file_path)
else:
raise
def ensure_dirs(dir_path, **kwargs):
"""Create a directory and call apply_permissions.
Returns True if a directory is created or the permissions needed to be
modified, and False otherwise.
This function's handling of EEXIST errors makes it useful for atomic
directory creation, in which multiple processes may be competing to
create the same directory.
"""
created_dir = False
try:
os.makedirs(dir_path)
created_dir = True
except OSError as oe:
func_call = "makedirs('%s')" % dir_path
if oe.errno in (errno.EEXIST,):
pass
else:
if os.path.isdir(dir_path):
# NOTE: DragonFly raises EPERM for makedir('/')
# and that is supposed to be ignored here.
# Also, sometimes mkdir raises EISDIR on FreeBSD
# and we want to ignore that too (bug #187518).
pass
elif oe.errno == errno.EPERM:
raise OperationNotPermitted(func_call)
elif oe.errno == errno.EACCES:
raise PermissionDenied(func_call)
elif oe.errno == errno.EROFS:
raise ReadOnlyFileSystem(func_call)
else:
raise
if kwargs:
perms_modified = apply_permissions(dir_path, **kwargs)
else:
perms_modified = False
return created_dir or perms_modified
def hardlink_lockfile(lockfilename,
max_wait=DeprecationWarning,
waiting_msg=None,
flags=0):
"""Does the NFS, hardlink shuffle to ensure locking on the disk.
We create a PRIVATE hardlink to the real lockfile, that is just a
placeholder on the disk.
If our file can 2 references, then we have the lock. :)
Otherwise we lather, rise, and repeat.
"""
if max_wait is not DeprecationWarning:
warnings.warn(
"The 'max_wait' parameter of "
"portage.locks.hardlink_lockfile() is now unused. Use "
"flags=os.O_NONBLOCK instead.",
DeprecationWarning,
stacklevel=2)
global _quiet
out = None
displayed_waiting_msg = False
preexisting = os.path.exists(lockfilename)
myhardlock = hardlock_name(lockfilename)
# Since Python 3.4, chown requires int type (no proxies).
portage_gid = int(portage.data.portage_gid)
# myhardlock must not exist prior to our link() call, and we can
# safely unlink it since its file name is unique to our PID
try:
os.unlink(myhardlock)
except OSError as e:
if e.errno in (errno.ENOENT, errno.ESTALE):
pass
else:
func_call = "unlink('%s')" % myhardlock
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
else:
raise
while True:
# create lockfilename if it doesn't exist yet
try:
myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660)
except OSError as e:
func_call = "open('%s')" % lockfilename
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
elif e.errno == ReadOnlyFileSystem.errno:
raise ReadOnlyFileSystem(func_call)
else:
raise
else:
myfd_st = None
try:
myfd_st = os.fstat(myfd)
if not preexisting:
# Don't chown the file if it is preexisting, since we
# want to preserve existing permissions in that case.
if portage.data.secpass >= 1 and myfd_st.st_gid != portage_gid:
os.fchown(myfd, -1, portage_gid)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
writemsg("%s: fchown('%s', -1, %d)\n" % \
(e, lockfilename, portage_gid), noiselevel=-1)
writemsg(_("Cannot chown a lockfile: '%s'\n") % \
lockfilename, noiselevel=-1)
writemsg(_("Group IDs of current user: %s\n") % \
" ".join(str(n) for n in os.getgroups()),
noiselevel=-1)
else:
# another process has removed the file, so we'll have
# to create it again
continue
finally:
os.close(myfd)
# If fstat shows more than one hardlink, then it's extremely
# unlikely that the following link call will result in a lock,
# so optimize away the wasteful link call and sleep or raise
# TryAgain.
if myfd_st is not None and myfd_st.st_nlink < 2:
try:
os.link(lockfilename, myhardlock)
except OSError as e:
func_call = "link('%s', '%s')" % (lockfilename, myhardlock)
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
elif e.errno in (errno.ESTALE, errno.ENOENT):
# another process has removed the file, so we'll have
# to create it again
continue
else:
raise
else:
if hardlink_is_mine(myhardlock, lockfilename):
if out is not None:
out.eend(os.EX_OK)
break
try:
os.unlink(myhardlock)
except OSError as e:
# This should not happen, since the file name of
# myhardlock is unique to our host and PID,
# and the above link() call succeeded.
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
raise FileNotFound(myhardlock)
if flags & os.O_NONBLOCK:
raise TryAgain(lockfilename)
if out is None and not _quiet:
out = portage.output.EOutput()
if out is not None and not displayed_waiting_msg:
displayed_waiting_msg = True
if waiting_msg is None:
waiting_msg = _("waiting for lock on %s\n") % lockfilename
out.ebegin(waiting_msg)
time.sleep(_HARDLINK_POLL_LATENCY)
return True
def lockfile(mypath,
wantnewlockfile=0,
unlinkfile=0,
waiting_msg=None,
flags=0):
"""
If wantnewlockfile is True then this creates a lockfile in the parent
directory as the file: '.' + basename + '.portage_lockfile'.
"""
if not mypath:
raise InvalidData(_("Empty path given"))
# Since Python 3.4, chown requires int type (no proxies).
portage_gid = int(portage.data.portage_gid)
# Support for file object or integer file descriptor parameters is
# deprecated due to ambiguity in whether or not it's safe to close
# the file descriptor, making it prone to "Bad file descriptor" errors
# or file descriptor leaks.
if isinstance(mypath, basestring) and mypath[-1] == '/':
mypath = mypath[:-1]
lockfilename_path = mypath
if hasattr(mypath, 'fileno'):
warnings.warn(
"portage.locks.lockfile() support for "
"file object parameters is deprecated. Use a file path instead.",
DeprecationWarning,
stacklevel=2)
lockfilename_path = getattr(mypath, 'name', None)
mypath = mypath.fileno()
if isinstance(mypath, int):
warnings.warn(
"portage.locks.lockfile() support for integer file "
"descriptor parameters is deprecated. Use a file path instead.",
DeprecationWarning,
stacklevel=2)
lockfilename = mypath
wantnewlockfile = 0
unlinkfile = 0
elif wantnewlockfile:
base, tail = os.path.split(mypath)
lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
lockfilename_path = lockfilename
unlinkfile = 1
else:
lockfilename = mypath
if isinstance(mypath, basestring):
if not os.path.exists(os.path.dirname(mypath)):
raise DirectoryNotFound(os.path.dirname(mypath))
preexisting = os.path.exists(lockfilename)
old_mask = os.umask(000)
try:
try:
myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660)
except OSError as e:
func_call = "open('%s')" % lockfilename
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
elif e.errno == ReadOnlyFileSystem.errno:
raise ReadOnlyFileSystem(func_call)
else:
raise
if not preexisting:
try:
if os.stat(lockfilename).st_gid != portage_gid:
os.chown(lockfilename, -1, portage_gid)
except OSError as e:
if e.errno in (errno.ENOENT, errno.ESTALE):
return lockfile(mypath,
wantnewlockfile=wantnewlockfile,
unlinkfile=unlinkfile,
waiting_msg=waiting_msg,
flags=flags)
else:
writemsg("%s: chown('%s', -1, %d)\n" % \
(e, lockfilename, portage_gid), noiselevel=-1)
writemsg(_("Cannot chown a lockfile: '%s'\n") % \
lockfilename, noiselevel=-1)
writemsg(_("Group IDs of current user: %s\n") % \
" ".join(str(n) for n in os.getgroups()),
noiselevel=-1)
finally:
os.umask(old_mask)
elif isinstance(mypath, int):
myfd = mypath
else:
raise ValueError(_("Unknown type passed in '%s': '%s'") % \
(type(mypath), mypath))
# try for a non-blocking lock, if it's held, throw a message
# we're waiting on lockfile and use a blocking attempt.
locking_method = portage._eintr_func_wrapper(_default_lock_fn)
try:
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
raise IOError(errno.ENOSYS, "Function not implemented")
locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError as e:
if not hasattr(e, "errno"):
raise
if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK):
# resource temp unavailable; eg, someone beat us to the lock.
if flags & os.O_NONBLOCK:
os.close(myfd)
raise TryAgain(mypath)
global _quiet
if _quiet:
out = None
else:
out = portage.output.EOutput()
if waiting_msg is None:
if isinstance(mypath, int):
waiting_msg = _("waiting for lock on fd %i") % myfd
else:
waiting_msg = _("waiting for lock on %s") % lockfilename
if out is not None:
out.ebegin(waiting_msg)
# try for the exclusive lock now.
enolock_msg_shown = False
while True:
try:
locking_method(myfd, fcntl.LOCK_EX)
except EnvironmentError as e:
if e.errno == errno.ENOLCK:
# This is known to occur on Solaris NFS (see
# bug #462694). Assume that the error is due
# to temporary exhaustion of record locks,
# and loop until one becomes available.
if not enolock_msg_shown:
enolock_msg_shown = True
if isinstance(mypath, int):
context_desc = _("Error while waiting "
"to lock fd %i") % myfd
else:
context_desc = _("Error while waiting "
"to lock '%s'") % lockfilename
writemsg("\n!!! %s: %s\n" % (context_desc, e),
noiselevel=-1)
time.sleep(_HARDLINK_POLL_LATENCY)
continue
if out is not None:
out.eend(1, str(e))
raise
else:
break
if out is not None:
out.eend(os.EX_OK)
elif e.errno in (errno.ENOSYS, ):
# We're not allowed to lock on this FS.
if not isinstance(lockfilename, int):
# If a file object was passed in, it's not safe
# to close the file descriptor because it may
# still be in use.
os.close(myfd)
lockfilename_path = _unicode_decode(lockfilename_path,
encoding=_encodings['fs'],
errors='strict')
if not isinstance(lockfilename_path, basestring):
raise
link_success = hardlink_lockfile(lockfilename_path,
waiting_msg=waiting_msg,
flags=flags)
if not link_success:
raise
lockfilename = lockfilename_path
locking_method = None
myfd = HARDLINK_FD
else:
raise
if isinstance(lockfilename, basestring) and \
myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
# The file was deleted on us... Keep trying to make one...
os.close(myfd)
writemsg(_("lockfile recurse\n"), 1)
lockfilename, myfd, unlinkfile, locking_method = lockfile(
mypath,
wantnewlockfile=wantnewlockfile,
unlinkfile=unlinkfile,
waiting_msg=waiting_msg,
flags=flags)
if myfd != HARDLINK_FD:
# FD_CLOEXEC is enabled by default in Python >=3.4.
if sys.hexversion < 0x3040000:
try:
fcntl.FD_CLOEXEC
except AttributeError:
pass
else:
fcntl.fcntl(
myfd, fcntl.F_SETFD,
fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
_open_fds.add(myfd)
writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1)
return (lockfilename, myfd, unlinkfile, locking_method)
def apply_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1,
stat_cached=None, follow_links=True):
"""Apply user, group, and mode bits to a file if the existing bits do not
already match. The default behavior is to force an exact match of mode
bits. When mask=0 is specified, mode bits on the target file are allowed
to be a superset of the mode argument (via logical OR). When mask>0, the
mode bits that the target file is allowed to have are restricted via
logical XOR.
Returns True if the permissions were modified and False otherwise."""
modified = False
if stat_cached is None:
try:
if follow_links:
stat_cached = os.stat(filename)
else:
stat_cached = os.lstat(filename)
except OSError as oe:
func_call = "stat('%s')" % filename
if oe.errno == errno.EPERM:
raise OperationNotPermitted(func_call)
elif oe.errno == errno.EACCES:
raise PermissionDenied(func_call)
elif oe.errno == errno.ENOENT:
raise FileNotFound(filename)
else:
raise
if (uid != -1 and uid != stat_cached.st_uid) or \
(gid != -1 and gid != stat_cached.st_gid):
try:
if follow_links:
os.chown(filename, uid, gid)
else:
portage.data.lchown(filename, uid, gid)
modified = True
except OSError as oe:
func_call = "chown('%s', %i, %i)" % (filename, uid, gid)
if oe.errno == errno.EPERM:
raise OperationNotPermitted(func_call)
elif oe.errno == errno.EACCES:
raise PermissionDenied(func_call)
elif oe.errno == errno.EROFS:
raise ReadOnlyFileSystem(func_call)
elif oe.errno == errno.ENOENT:
raise FileNotFound(filename)
else:
raise
new_mode = -1
st_mode = stat_cached.st_mode & 0o7777 # protect from unwanted bits
if mask >= 0:
if mode == -1:
mode = 0 # Don't add any mode bits when mode is unspecified.
else:
mode = mode & 0o7777
if (mode & st_mode != mode) or \
((mask ^ st_mode) & st_mode != st_mode):
new_mode = mode | st_mode
new_mode = (mask ^ new_mode) & new_mode
elif mode != -1:
mode = mode & 0o7777 # protect from unwanted bits
if mode != st_mode:
new_mode = mode
# The chown system call may clear S_ISUID and S_ISGID
# bits, so those bits are restored if necessary.
if modified and new_mode == -1 and \
(st_mode & stat.S_ISUID or st_mode & stat.S_ISGID):
if mode == -1:
new_mode = st_mode
else:
mode = mode & 0o7777
if mask >= 0:
new_mode = mode | st_mode
new_mode = (mask ^ new_mode) & new_mode
else:
new_mode = mode
if not (new_mode & stat.S_ISUID or new_mode & stat.S_ISGID):
new_mode = -1
if not follow_links and stat.S_ISLNK(stat_cached.st_mode):
# Mode doesn't matter for symlinks.
new_mode = -1
if new_mode != -1:
try:
os.chmod(filename, new_mode)
modified = True
except OSError as oe:
func_call = "chmod('%s', %s)" % (filename, oct(new_mode))
if oe.errno == errno.EPERM:
raise OperationNotPermitted(func_call)
elif oe.errno == errno.EACCES:
raise PermissionDenied(func_call)
elif oe.errno == errno.EROFS:
raise ReadOnlyFileSystem(func_call)
elif oe.errno == errno.ENOENT:
raise FileNotFound(filename)
raise
return modified