def _lockfile_was_removed(lock_fd, lock_path): """ Check if lock_fd still refers to a file located at lock_path, since the file may have been removed by a concurrent process that held the lock earlier. This implementation includes support for NFS, where stat is not reliable for removed files due to the default file attribute cache behavior ('ac' mount option). @param lock_fd: an open file descriptor for a lock file @type lock_fd: int @param lock_path: path of lock file @type lock_path: str @rtype: bool @return: True if lock_path exists and corresponds to lock_fd, False otherwise """ try: fstat_st = os.fstat(lock_fd) except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): _raise_exc(e) return True # Since stat is not reliable for removed files on NFS with the default # file attribute cache behavior ('ac' mount option), create a temporary # hardlink in order to prove that the file path exists on the NFS server. hardlink_path = hardlock_name(lock_path) try: os.unlink(hardlink_path) except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): _raise_exc(e) try: try: os.link(lock_path, hardlink_path) except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): _raise_exc(e) return True hardlink_stat = os.stat(hardlink_path) if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev: return True finally: try: os.unlink(hardlink_path) except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): _raise_exc(e) return False
def prepare_build_dirs(myroot=None, settings=None, cleanup=False): """ The myroot parameter is ignored. """ myroot = None if settings is None: raise TypeError("settings argument is required") mysettings = settings clean_dirs = [mysettings["HOME"]] # We enable cleanup when we want to make sure old cruft (such as the old # environment) doesn't interfere with the current phase. if cleanup and 'keeptemp' not in mysettings.features: clean_dirs.append(mysettings["T"]) for clean_dir in clean_dirs: try: shutil.rmtree(clean_dir) except OSError as oe: if errno.ENOENT == oe.errno: pass elif errno.EPERM == oe.errno: writemsg("%s\n" % oe, noiselevel=-1) writemsg(_("Operation Not Permitted: rmtree('%s')\n") % \ clean_dir, noiselevel=-1) return 1 else: # Wrap with PermissionDenied if appropriate, so that callers # display a short error message without a traceback. _raise_exc(oe) def makedirs(dir_path): try: os.makedirs(dir_path) except OSError as oe: if errno.EEXIST == oe.errno: pass elif errno.EPERM == oe.errno: writemsg("%s\n" % oe, noiselevel=-1) writemsg(_("Operation Not Permitted: makedirs('%s')\n") % \ dir_path, noiselevel=-1) return False else: raise return True mysettings["PKG_LOGDIR"] = os.path.join(mysettings["T"], "logging") mydirs = [os.path.dirname(mysettings["PORTAGE_BUILDDIR"])] mydirs.append(os.path.dirname(mydirs[-1])) try: for mydir in mydirs: ensure_dirs(mydir) try: apply_secpass_permissions(mydir, gid=portage_gid, uid=portage_uid, mode=0o700, mask=0) except PortageException: if not os.path.isdir(mydir): raise for dir_key in ("PORTAGE_BUILDDIR", "HOME", "PKG_LOGDIR", "T"): ensure_dirs(mysettings[dir_key], mode=0o755) apply_secpass_permissions(mysettings[dir_key], uid=portage_uid, gid=portage_gid) except PermissionDenied as e: writemsg(_("Permission Denied: %s\n") % str(e), noiselevel=-1) return 1 except OperationNotPermitted as e: writemsg(_("Operation Not Permitted: %s\n") % str(e), noiselevel=-1) return 1 except FileNotFound as e: writemsg(_("File Not Found: '%s'\n") % str(e), noiselevel=-1) return 1 # Reset state for things like noauto and keepwork in FEATURES. for x in ('.die_hooks',): try: os.unlink(os.path.join(mysettings['PORTAGE_BUILDDIR'], x)) except OSError: pass _prepare_workdir(mysettings) if mysettings.get("EBUILD_PHASE") not in ("info", "fetch", "pretend"): # Avoid spurious permissions adjustments when fetching with # a temporary PORTAGE_TMPDIR setting (for fetchonly). _prepare_features_dirs(mysettings)
def _lockfile_was_removed(lock_fd, lock_path): """ Check if lock_fd still refers to a file located at lock_path, since the file may have been removed by a concurrent process that held the lock earlier. This implementation includes support for NFS, where stat is not reliable for removed files due to the default file attribute cache behavior ('ac' mount option). @param lock_fd: an open file descriptor for a lock file @type lock_fd: int @param lock_path: path of lock file @type lock_path: str @rtype: bool @return: True if lock_path exists and corresponds to lock_fd, False otherwise """ try: fstat_st = os.fstat(lock_fd) except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): _raise_exc(e) return True # Since stat is not reliable for removed files on NFS with the default # file attribute cache behavior ('ac' mount option), create a temporary # hardlink in order to prove that the file path exists on the NFS server. hardlink_path = hardlock_name(lock_path) try: os.unlink(hardlink_path) except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): _raise_exc(e) try: try: os.link(lock_path, hardlink_path) except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): _raise_exc(e) return True hardlink_stat = os.stat(hardlink_path) if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev: # Create another hardlink in order to detect whether or not # hardlink inode numbers are expected to match. For example, # inode numbers are not expected to match for sshfs. inode_test = hardlink_path + '-inode-test' try: os.unlink(inode_test) except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): _raise_exc(e) try: os.link(hardlink_path, inode_test) except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): _raise_exc(e) return True else: if not os.path.samefile(hardlink_path, inode_test): # This implies that inode numbers are not expected # to match for this file system, so use a simple # stat call to detect if lock_path has been removed. return not os.path.exists(lock_path) finally: try: os.unlink(inode_test) except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): _raise_exc(e) return True finally: try: os.unlink(hardlink_path) except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): _raise_exc(e) return False
def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False, waiting_msg=None, flags=0): """ Acquire a lock on mypath, without retry. Return None if the lockfile was removed by previous lock holder (caller must retry). @param mypath: lock file path @type mypath: str @param wantnewlockfile: use a separate new lock file @type wantnewlockfile: bool @param unlinkfile: remove lock file prior to unlock @type unlinkfile: bool @param waiting_msg: message to show before blocking @type waiting_msg: str @param flags: lock flags (only supports os.O_NONBLOCK) @type flags: int @rtype: bool @return: unlockfile tuple on success, None if retry is needed """ if not mypath: raise InvalidData(_("Empty path given")) # Since Python 3.4, chown requires int type (no proxies). portage_gid = int(portage.data.portage_gid) # Support for file object or integer file descriptor parameters is # deprecated due to ambiguity in whether or not it's safe to close # the file descriptor, making it prone to "Bad file descriptor" errors # or file descriptor leaks. if isinstance(mypath, basestring) and mypath[-1] == '/': mypath = mypath[:-1] lockfilename_path = mypath if hasattr(mypath, 'fileno'): warnings.warn( "portage.locks.lockfile() support for " "file object parameters is deprecated. Use a file path instead.", DeprecationWarning, stacklevel=2) lockfilename_path = getattr(mypath, 'name', None) mypath = mypath.fileno() if isinstance(mypath, int): warnings.warn( "portage.locks.lockfile() support for integer file " "descriptor parameters is deprecated. Use a file path instead.", DeprecationWarning, stacklevel=2) lockfilename = mypath wantnewlockfile = 0 unlinkfile = 0 elif wantnewlockfile: base, tail = os.path.split(mypath) lockfilename = os.path.join(base, "." + tail + ".portage_lockfile") lockfilename_path = lockfilename unlinkfile = 1 else: lockfilename = mypath if isinstance(mypath, basestring): if not os.path.exists(os.path.dirname(mypath)): raise DirectoryNotFound(os.path.dirname(mypath)) preexisting = os.path.exists(lockfilename) old_mask = os.umask(000) try: while True: try: myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE) and os.path.isdir( os.path.dirname(lockfilename)): # Retry required for NFS (see bug 636798). continue else: _raise_exc(e) else: break if not preexisting: try: if portage.data.secpass >= 1 and os.stat( lockfilename).st_gid != portage_gid: os.chown(lockfilename, -1, portage_gid) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE): os.close(myfd) return None else: writemsg("%s: chown('%s', -1, %d)\n" % \ (e, lockfilename, portage_gid), noiselevel=-1) writemsg(_("Cannot chown a lockfile: '%s'\n") % \ lockfilename, noiselevel=-1) writemsg(_("Group IDs of current user: %s\n") % \ " ".join(str(n) for n in os.getgroups()), noiselevel=-1) finally: os.umask(old_mask) elif isinstance(mypath, int): myfd = mypath else: raise ValueError(_("Unknown type passed in '%s': '%s'") % \ (type(mypath), mypath)) # try for a non-blocking lock, if it's held, throw a message # we're waiting on lockfile and use a blocking attempt. locking_method = portage._eintr_func_wrapper(_get_lock_fn()) try: if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: raise IOError(errno.ENOSYS, "Function not implemented") locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: if not hasattr(e, "errno"): raise if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK): # resource temp unavailable; eg, someone beat us to the lock. if flags & os.O_NONBLOCK: os.close(myfd) raise TryAgain(mypath) global _quiet if _quiet: out = None else: out = portage.output.EOutput() if waiting_msg is None: if isinstance(mypath, int): waiting_msg = _("waiting for lock on fd %i") % myfd else: waiting_msg = _("waiting for lock on %s") % lockfilename if out is not None: out.ebegin(waiting_msg) # try for the exclusive lock now. enolock_msg_shown = False while True: try: locking_method(myfd, fcntl.LOCK_EX) except EnvironmentError as e: if e.errno == errno.ENOLCK: # This is known to occur on Solaris NFS (see # bug #462694). Assume that the error is due # to temporary exhaustion of record locks, # and loop until one becomes available. if not enolock_msg_shown: enolock_msg_shown = True if isinstance(mypath, int): context_desc = _("Error while waiting " "to lock fd %i") % myfd else: context_desc = _("Error while waiting " "to lock '%s'") % lockfilename writemsg("\n!!! %s: %s\n" % (context_desc, e), noiselevel=-1) time.sleep(_HARDLINK_POLL_LATENCY) continue if out is not None: out.eend(1, str(e)) raise else: break if out is not None: out.eend(os.EX_OK) elif e.errno in (errno.ENOSYS, ): # We're not allowed to lock on this FS. if not isinstance(lockfilename, int): # If a file object was passed in, it's not safe # to close the file descriptor because it may # still be in use. os.close(myfd) lockfilename_path = _unicode_decode(lockfilename_path, encoding=_encodings['fs'], errors='strict') if not isinstance(lockfilename_path, basestring): raise link_success = hardlink_lockfile(lockfilename_path, waiting_msg=waiting_msg, flags=flags) if not link_success: raise lockfilename = lockfilename_path locking_method = None myfd = HARDLINK_FD else: raise if isinstance(lockfilename, basestring) and myfd != HARDLINK_FD and unlinkfile: try: removed = _lockfile_was_removed(myfd, lockfilename) except Exception: # Do not leak the file descriptor here. os.close(myfd) raise else: if removed: # Removed by previous lock holder... Caller will retry... os.close(myfd) return None if myfd != HARDLINK_FD: # FD_CLOEXEC is enabled by default in Python >=3.4. if sys.hexversion < 0x3040000: try: fcntl.FD_CLOEXEC except AttributeError: pass else: fcntl.fcntl( myfd, fcntl.F_SETFD, fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) _open_fds.add(myfd) writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1) return (lockfilename, myfd, unlinkfile, locking_method)
def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False, waiting_msg=None, flags=0): """ Acquire a lock on mypath, without retry. Return None if the lockfile was removed by previous lock holder (caller must retry). @param mypath: lock file path @type mypath: str @param wantnewlockfile: use a separate new lock file @type wantnewlockfile: bool @param unlinkfile: remove lock file prior to unlock @type unlinkfile: bool @param waiting_msg: message to show before blocking @type waiting_msg: str @param flags: lock flags (only supports os.O_NONBLOCK) @type flags: int @rtype: bool @return: unlockfile tuple on success, None if retry is needed """ if not mypath: raise InvalidData(_("Empty path given")) # Since Python 3.4, chown requires int type (no proxies). portage_gid = int(portage.data.portage_gid) # Support for file object or integer file descriptor parameters is # deprecated due to ambiguity in whether or not it's safe to close # the file descriptor, making it prone to "Bad file descriptor" errors # or file descriptor leaks. if isinstance(mypath, basestring) and mypath[-1] == '/': mypath = mypath[:-1] lockfilename_path = mypath if hasattr(mypath, 'fileno'): warnings.warn("portage.locks.lockfile() support for " "file object parameters is deprecated. Use a file path instead.", DeprecationWarning, stacklevel=2) lockfilename_path = getattr(mypath, 'name', None) mypath = mypath.fileno() if isinstance(mypath, int): warnings.warn("portage.locks.lockfile() support for integer file " "descriptor parameters is deprecated. Use a file path instead.", DeprecationWarning, stacklevel=2) lockfilename = mypath wantnewlockfile = 0 unlinkfile = 0 elif wantnewlockfile: base, tail = os.path.split(mypath) lockfilename = os.path.join(base, "." + tail + ".portage_lockfile") lockfilename_path = lockfilename unlinkfile = 1 else: lockfilename = mypath if isinstance(mypath, basestring): if not os.path.exists(os.path.dirname(mypath)): raise DirectoryNotFound(os.path.dirname(mypath)) preexisting = os.path.exists(lockfilename) old_mask = os.umask(000) try: while True: try: myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE) and os.path.isdir(os.path.dirname(lockfilename)): # Retry required for NFS (see bug 636798). continue else: _raise_exc(e) else: break if not preexisting: try: if portage.data.secpass >= 1 and os.stat(lockfilename).st_gid != portage_gid: os.chown(lockfilename, -1, portage_gid) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE): return lockfile(mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile, waiting_msg=waiting_msg, flags=flags) else: writemsg("%s: chown('%s', -1, %d)\n" % \ (e, lockfilename, portage_gid), noiselevel=-1) writemsg(_("Cannot chown a lockfile: '%s'\n") % \ lockfilename, noiselevel=-1) writemsg(_("Group IDs of current user: %s\n") % \ " ".join(str(n) for n in os.getgroups()), noiselevel=-1) finally: os.umask(old_mask) elif isinstance(mypath, int): myfd = mypath else: raise ValueError(_("Unknown type passed in '%s': '%s'") % \ (type(mypath), mypath)) # try for a non-blocking lock, if it's held, throw a message # we're waiting on lockfile and use a blocking attempt. locking_method = portage._eintr_func_wrapper(_get_lock_fn()) try: if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: raise IOError(errno.ENOSYS, "Function not implemented") locking_method(myfd, fcntl.LOCK_EX|fcntl.LOCK_NB) except IOError as e: if not hasattr(e, "errno"): raise if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK): # resource temp unavailable; eg, someone beat us to the lock. if flags & os.O_NONBLOCK: os.close(myfd) raise TryAgain(mypath) global _quiet if _quiet: out = None else: out = portage.output.EOutput() if waiting_msg is None: if isinstance(mypath, int): waiting_msg = _("waiting for lock on fd %i") % myfd else: waiting_msg = _("waiting for lock on %s") % lockfilename if out is not None: out.ebegin(waiting_msg) # try for the exclusive lock now. enolock_msg_shown = False while True: try: locking_method(myfd, fcntl.LOCK_EX) except EnvironmentError as e: if e.errno == errno.ENOLCK: # This is known to occur on Solaris NFS (see # bug #462694). Assume that the error is due # to temporary exhaustion of record locks, # and loop until one becomes available. if not enolock_msg_shown: enolock_msg_shown = True if isinstance(mypath, int): context_desc = _("Error while waiting " "to lock fd %i") % myfd else: context_desc = _("Error while waiting " "to lock '%s'") % lockfilename writemsg("\n!!! %s: %s\n" % (context_desc, e), noiselevel=-1) time.sleep(_HARDLINK_POLL_LATENCY) continue if out is not None: out.eend(1, str(e)) raise else: break if out is not None: out.eend(os.EX_OK) elif e.errno in (errno.ENOSYS,): # We're not allowed to lock on this FS. if not isinstance(lockfilename, int): # If a file object was passed in, it's not safe # to close the file descriptor because it may # still be in use. os.close(myfd) lockfilename_path = _unicode_decode(lockfilename_path, encoding=_encodings['fs'], errors='strict') if not isinstance(lockfilename_path, basestring): raise link_success = hardlink_lockfile(lockfilename_path, waiting_msg=waiting_msg, flags=flags) if not link_success: raise lockfilename = lockfilename_path locking_method = None myfd = HARDLINK_FD else: raise if isinstance(lockfilename, basestring) and myfd != HARDLINK_FD and unlinkfile: try: removed = _lockfile_was_removed(myfd, lockfilename) except Exception: # Do not leak the file descriptor here. os.close(myfd) raise else: if removed: # Removed by previous lock holder... Caller will retry... os.close(myfd) return None if myfd != HARDLINK_FD: # FD_CLOEXEC is enabled by default in Python >=3.4. if sys.hexversion < 0x3040000: try: fcntl.FD_CLOEXEC except AttributeError: pass else: fcntl.fcntl(myfd, fcntl.F_SETFD, fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) _open_fds.add(myfd) writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1) return (lockfilename, myfd, unlinkfile, locking_method)