def __init__(self, fd, fstat_result, path): self.fd = fd self.inode_key = (fstat_result.st_dev, fstat_result.st_ino) if self.inode_key in _open_inodes: # This means that the lock is already held by the current # process, so the caller will have to try again. This case # is encountered with the default fcntl.lockf function, and # with the alternative fcntl.flock function TryAgain is # raised earlier. os.close(fd) raise TryAgain(path) _open_fds[fd] = self _open_inodes[self.inode_key] = self
def hardlink_lockfile(lockfilename, max_wait=DeprecationWarning, waiting_msg=None, flags=0): """Does the NFS, hardlink shuffle to ensure locking on the disk. We create a PRIVATE hardlink to the real lockfile, that is just a placeholder on the disk. If our file can 2 references, then we have the lock. :) Otherwise we lather, rise, and repeat. """ if max_wait is not DeprecationWarning: warnings.warn( "The 'max_wait' parameter of " "portage.locks.hardlink_lockfile() is now unused. Use " "flags=os.O_NONBLOCK instead.", DeprecationWarning, stacklevel=2) global _quiet out = None displayed_waiting_msg = False preexisting = os.path.exists(lockfilename) myhardlock = hardlock_name(lockfilename) # Since Python 3.4, chown requires int type (no proxies). portage_gid = int(portage.data.portage_gid) # myhardlock must not exist prior to our link() call, and we can # safely unlink it since its file name is unique to our PID try: os.unlink(myhardlock) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE): pass else: func_call = "unlink('%s')" % myhardlock if e.errno == OperationNotPermitted.errno: raise OperationNotPermitted(func_call) elif e.errno == PermissionDenied.errno: raise PermissionDenied(func_call) else: raise while True: # create lockfilename if it doesn't exist yet try: myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660) except OSError as e: func_call = "open('%s')" % lockfilename if e.errno == OperationNotPermitted.errno: raise OperationNotPermitted(func_call) elif e.errno == PermissionDenied.errno: raise PermissionDenied(func_call) elif e.errno == ReadOnlyFileSystem.errno: raise ReadOnlyFileSystem(func_call) else: raise else: myfd_st = None try: myfd_st = os.fstat(myfd) if not preexisting: # Don't chown the file if it is preexisting, since we # want to preserve existing permissions in that case. if portage.data.secpass >= 1 and myfd_st.st_gid != portage_gid: os.fchown(myfd, -1, portage_gid) except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): writemsg("%s: fchown('%s', -1, %d)\n" % \ (e, lockfilename, portage_gid), noiselevel=-1) writemsg(_("Cannot chown a lockfile: '%s'\n") % \ lockfilename, noiselevel=-1) writemsg(_("Group IDs of current user: %s\n") % \ " ".join(str(n) for n in os.getgroups()), noiselevel=-1) else: # another process has removed the file, so we'll have # to create it again continue finally: os.close(myfd) # If fstat shows more than one hardlink, then it's extremely # unlikely that the following link call will result in a lock, # so optimize away the wasteful link call and sleep or raise # TryAgain. if myfd_st is not None and myfd_st.st_nlink < 2: try: os.link(lockfilename, myhardlock) except OSError as e: func_call = "link('%s', '%s')" % (lockfilename, myhardlock) if e.errno == OperationNotPermitted.errno: raise OperationNotPermitted(func_call) elif e.errno == PermissionDenied.errno: raise PermissionDenied(func_call) elif e.errno in (errno.ESTALE, errno.ENOENT): # another process has removed the file, so we'll have # to create it again continue else: raise else: if hardlink_is_mine(myhardlock, lockfilename): if out is not None: out.eend(os.EX_OK) break try: os.unlink(myhardlock) except OSError as e: # This should not happen, since the file name of # myhardlock is unique to our host and PID, # and the above link() call succeeded. if e.errno not in (errno.ENOENT, errno.ESTALE): raise raise FileNotFound(myhardlock) if flags & os.O_NONBLOCK: raise TryAgain(lockfilename) if out is None and not _quiet: out = portage.output.EOutput() if out is not None and not displayed_waiting_msg: displayed_waiting_msg = True if waiting_msg is None: waiting_msg = _("waiting for lock on %s\n") % lockfilename out.ebegin(waiting_msg) time.sleep(_HARDLINK_POLL_LATENCY) return True
def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False, waiting_msg=None, flags=0): """ Acquire a lock on mypath, without retry. Return None if the lockfile was removed by previous lock holder (caller must retry). @param mypath: lock file path @type mypath: str @param wantnewlockfile: use a separate new lock file @type wantnewlockfile: bool @param unlinkfile: remove lock file prior to unlock @type unlinkfile: bool @param waiting_msg: message to show before blocking @type waiting_msg: str @param flags: lock flags (only supports os.O_NONBLOCK) @type flags: int @rtype: bool @return: unlockfile tuple on success, None if retry is needed """ if not mypath: raise InvalidData(_("Empty path given")) # Since Python 3.4, chown requires int type (no proxies). portage_gid = int(portage.data.portage_gid) # Support for file object or integer file descriptor parameters is # deprecated due to ambiguity in whether or not it's safe to close # the file descriptor, making it prone to "Bad file descriptor" errors # or file descriptor leaks. if isinstance(mypath, basestring) and mypath[-1] == '/': mypath = mypath[:-1] lockfilename_path = mypath if hasattr(mypath, 'fileno'): warnings.warn( "portage.locks.lockfile() support for " "file object parameters is deprecated. Use a file path instead.", DeprecationWarning, stacklevel=2) lockfilename_path = getattr(mypath, 'name', None) mypath = mypath.fileno() if isinstance(mypath, int): warnings.warn( "portage.locks.lockfile() support for integer file " "descriptor parameters is deprecated. Use a file path instead.", DeprecationWarning, stacklevel=2) lockfilename = mypath wantnewlockfile = 0 unlinkfile = 0 elif wantnewlockfile: base, tail = os.path.split(mypath) lockfilename = os.path.join(base, "." + tail + ".portage_lockfile") lockfilename_path = lockfilename unlinkfile = 1 else: lockfilename = mypath if isinstance(mypath, basestring): if not os.path.exists(os.path.dirname(mypath)): raise DirectoryNotFound(os.path.dirname(mypath)) preexisting = os.path.exists(lockfilename) old_mask = os.umask(000) try: while True: try: myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE) and os.path.isdir( os.path.dirname(lockfilename)): # Retry required for NFS (see bug 636798). continue else: _raise_exc(e) else: break if not preexisting: try: if portage.data.secpass >= 1 and os.stat( lockfilename).st_gid != portage_gid: os.chown(lockfilename, -1, portage_gid) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE): os.close(myfd) return None else: writemsg("%s: chown('%s', -1, %d)\n" % \ (e, lockfilename, portage_gid), noiselevel=-1) writemsg(_("Cannot chown a lockfile: '%s'\n") % \ lockfilename, noiselevel=-1) writemsg(_("Group IDs of current user: %s\n") % \ " ".join(str(n) for n in os.getgroups()), noiselevel=-1) finally: os.umask(old_mask) elif isinstance(mypath, int): myfd = mypath else: raise ValueError(_("Unknown type passed in '%s': '%s'") % \ (type(mypath), mypath)) # try for a non-blocking lock, if it's held, throw a message # we're waiting on lockfile and use a blocking attempt. locking_method = portage._eintr_func_wrapper(_get_lock_fn()) try: if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: raise IOError(errno.ENOSYS, "Function not implemented") locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: if not hasattr(e, "errno"): raise if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK): # resource temp unavailable; eg, someone beat us to the lock. if flags & os.O_NONBLOCK: os.close(myfd) raise TryAgain(mypath) global _quiet if _quiet: out = None else: out = portage.output.EOutput() if waiting_msg is None: if isinstance(mypath, int): waiting_msg = _("waiting for lock on fd %i") % myfd else: waiting_msg = _("waiting for lock on %s") % lockfilename if out is not None: out.ebegin(waiting_msg) # try for the exclusive lock now. enolock_msg_shown = False while True: try: locking_method(myfd, fcntl.LOCK_EX) except EnvironmentError as e: if e.errno == errno.ENOLCK: # This is known to occur on Solaris NFS (see # bug #462694). Assume that the error is due # to temporary exhaustion of record locks, # and loop until one becomes available. if not enolock_msg_shown: enolock_msg_shown = True if isinstance(mypath, int): context_desc = _("Error while waiting " "to lock fd %i") % myfd else: context_desc = _("Error while waiting " "to lock '%s'") % lockfilename writemsg("\n!!! %s: %s\n" % (context_desc, e), noiselevel=-1) time.sleep(_HARDLINK_POLL_LATENCY) continue if out is not None: out.eend(1, str(e)) raise else: break if out is not None: out.eend(os.EX_OK) elif e.errno in (errno.ENOSYS, ): # We're not allowed to lock on this FS. if not isinstance(lockfilename, int): # If a file object was passed in, it's not safe # to close the file descriptor because it may # still be in use. os.close(myfd) lockfilename_path = _unicode_decode(lockfilename_path, encoding=_encodings['fs'], errors='strict') if not isinstance(lockfilename_path, basestring): raise link_success = hardlink_lockfile(lockfilename_path, waiting_msg=waiting_msg, flags=flags) if not link_success: raise lockfilename = lockfilename_path locking_method = None myfd = HARDLINK_FD else: raise if isinstance(lockfilename, basestring) and myfd != HARDLINK_FD and unlinkfile: try: removed = _lockfile_was_removed(myfd, lockfilename) except Exception: # Do not leak the file descriptor here. os.close(myfd) raise else: if removed: # Removed by previous lock holder... Caller will retry... os.close(myfd) return None if myfd != HARDLINK_FD: # FD_CLOEXEC is enabled by default in Python >=3.4. if sys.hexversion < 0x3040000: try: fcntl.FD_CLOEXEC except AttributeError: pass else: fcntl.fcntl( myfd, fcntl.F_SETFD, fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) _open_fds.add(myfd) writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1) return (lockfilename, myfd, unlinkfile, locking_method)
def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=None, flags=0): """ If wantnewlockfile is True then this creates a lockfile in the parent directory as the file: '.' + basename + '.portage_lockfile'. """ if not mypath: raise InvalidData(_("Empty path given")) # Support for file object or integer file descriptor parameters is # deprecated due to ambiguity in whether or not it's safe to close # the file descriptor, making it prone to "Bad file descriptor" errors # or file descriptor leaks. if isinstance(mypath, basestring) and mypath[-1] == '/': mypath = mypath[:-1] lockfilename_path = mypath if hasattr(mypath, 'fileno'): warnings.warn( "portage.locks.lockfile() support for " "file object parameters is deprecated. Use a file path instead.", DeprecationWarning, stacklevel=2) lockfilename_path = getattr(mypath, 'name', None) mypath = mypath.fileno() if isinstance(mypath, int): warnings.warn( "portage.locks.lockfile() support for integer file " "descriptor parameters is deprecated. Use a file path instead.", DeprecationWarning, stacklevel=2) lockfilename = mypath wantnewlockfile = 0 unlinkfile = 0 elif wantnewlockfile: base, tail = os.path.split(mypath) lockfilename = os.path.join(base, "." + tail + ".portage_lockfile") lockfilename_path = lockfilename unlinkfile = 1 else: lockfilename = mypath if isinstance(mypath, basestring): if not os.path.exists(os.path.dirname(mypath)): raise DirectoryNotFound(os.path.dirname(mypath)) preexisting = os.path.exists(lockfilename) old_mask = os.umask(000) try: try: myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660) except OSError as e: func_call = "open('%s')" % lockfilename if e.errno == OperationNotPermitted.errno: raise OperationNotPermitted(func_call) elif e.errno == PermissionDenied.errno: raise PermissionDenied(func_call) else: raise if not preexisting: try: if os.stat(lockfilename).st_gid != portage_gid: os.chown(lockfilename, -1, portage_gid) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE): return lockfile(mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile, waiting_msg=waiting_msg, flags=flags) else: writemsg("%s: chown('%s', -1, %d)\n" % \ (e, lockfilename, portage_gid), noiselevel=-1) writemsg(_("Cannot chown a lockfile: '%s'\n") % \ lockfilename, noiselevel=-1) writemsg(_("Group IDs of current user: %s\n") % \ " ".join(str(n) for n in os.getgroups()), noiselevel=-1) finally: os.umask(old_mask) elif isinstance(mypath, int): myfd = mypath else: raise ValueError(_("Unknown type passed in '%s': '%s'") % \ (type(mypath), mypath)) # try for a non-blocking lock, if it's held, throw a message # we're waiting on lockfile and use a blocking attempt. locking_method = _default_lock_fn try: if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: raise IOError(errno.ENOSYS, "Function not implemented") locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: if not hasattr(e, "errno"): raise if e.errno in (errno.EACCES, errno.EAGAIN): # resource temp unavailable; eg, someone beat us to the lock. if flags & os.O_NONBLOCK: os.close(myfd) raise TryAgain(mypath) global _quiet if _quiet: out = None else: out = portage.output.EOutput() if waiting_msg is None: if isinstance(mypath, int): waiting_msg = _("waiting for lock on fd %i") % myfd else: waiting_msg = _("waiting for lock on %s\n") % lockfilename if out is not None: out.ebegin(waiting_msg) # try for the exclusive lock now. try: locking_method(myfd, fcntl.LOCK_EX) except EnvironmentError as e: if out is not None: out.eend(1, str(e)) raise if out is not None: out.eend(os.EX_OK) elif e.errno in (errno.ENOSYS, errno.ENOLCK): # We're not allowed to lock on this FS. if not isinstance(lockfilename, int): # If a file object was passed in, it's not safe # to close the file descriptor because it may # still be in use. os.close(myfd) lockfilename_path = _unicode_decode(lockfilename_path, encoding=_encodings['fs'], errors='strict') if not isinstance(lockfilename_path, basestring): raise link_success = hardlink_lockfile(lockfilename_path, waiting_msg=waiting_msg, flags=flags) if not link_success: raise lockfilename = lockfilename_path locking_method = None myfd = HARDLINK_FD else: raise if isinstance(lockfilename, basestring) and \ myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0: # The file was deleted on us... Keep trying to make one... os.close(myfd) writemsg(_("lockfile recurse\n"), 1) lockfilename, myfd, unlinkfile, locking_method = lockfile( mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile, waiting_msg=waiting_msg, flags=flags) if myfd != HARDLINK_FD: _open_fds.add(myfd) writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1) return (lockfilename, myfd, unlinkfile, locking_method)
def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=None, flags=0): """ If wantnewlockfile is True then this creates a lockfile in the parent directory as the file: '.' + basename + '.portage_lockfile'. """ import fcntl if not mypath: raise InvalidData(_("Empty path given")) if isinstance(mypath, basestring) and mypath[-1] == '/': mypath = mypath[:-1] if hasattr(mypath, 'fileno'): mypath = mypath.fileno() if isinstance(mypath, int): lockfilename = mypath wantnewlockfile = 0 unlinkfile = 0 elif wantnewlockfile: base, tail = os.path.split(mypath) lockfilename = os.path.join(base, "." + tail + ".portage_lockfile") del base, tail unlinkfile = 1 else: lockfilename = mypath if isinstance(mypath, basestring): if not os.path.exists(os.path.dirname(mypath)): raise DirectoryNotFound(os.path.dirname(mypath)) preexisting = os.path.exists(lockfilename) old_mask = os.umask(000) try: try: myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660) except OSError as e: func_call = "open('%s')" % lockfilename if e.errno == OperationNotPermitted.errno: raise OperationNotPermitted(func_call) elif e.errno == PermissionDenied.errno: raise PermissionDenied(func_call) else: raise if not preexisting: try: if os.stat(lockfilename).st_gid != portage_gid: os.chown(lockfilename, -1, portage_gid) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE): return lockfile(mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile, waiting_msg=waiting_msg, flags=flags) else: writemsg("%s: chown('%s', -1, %d)\n" % \ (e, lockfilename, portage_gid), noiselevel=-1) writemsg(_("Cannot chown a lockfile: '%s'\n") % \ lockfilename, noiselevel=-1) writemsg(_("Group IDs of current user: %s\n") % \ " ".join(str(n) for n in os.getgroups()), noiselevel=-1) finally: os.umask(old_mask) elif isinstance(mypath, int): myfd = mypath else: raise ValueError(_("Unknown type passed in '%s': '%s'") % \ (type(mypath), mypath)) # try for a non-blocking lock, if it's held, throw a message # we're waiting on lockfile and use a blocking attempt. locking_method = fcntl.lockf try: fcntl.lockf(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: if "errno" not in dir(e): raise if e.errno in (errno.EACCES, errno.EAGAIN): # resource temp unavailable; eg, someone beat us to the lock. if flags & os.O_NONBLOCK: raise TryAgain(mypath) global _quiet out = EOutput() out.quiet = _quiet if waiting_msg is None: if isinstance(mypath, int): waiting_msg = _("waiting for lock on fd %i") % myfd else: waiting_msg = _("waiting for lock on %s\n") % lockfilename out.ebegin(waiting_msg) # try for the exclusive lock now. try: fcntl.lockf(myfd, fcntl.LOCK_EX) except EnvironmentError as e: out.eend(1, str(e)) raise out.eend(os.EX_OK) elif e.errno == errno.ENOLCK: # We're not allowed to lock on this FS. os.close(myfd) link_success = False if lockfilename == str(lockfilename): if wantnewlockfile: try: if os.stat(lockfilename)[stat.ST_NLINK] == 1: os.unlink(lockfilename) except OSError: pass link_success = hardlink_lockfile(lockfilename) if not link_success: raise locking_method = None myfd = HARDLINK_FD else: raise if isinstance(lockfilename, basestring) and \ myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0: # The file was deleted on us... Keep trying to make one... os.close(myfd) writemsg(_("lockfile recurse\n"), 1) lockfilename, myfd, unlinkfile, locking_method = lockfile( mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile, waiting_msg=waiting_msg, flags=flags) writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1) return (lockfilename, myfd, unlinkfile, locking_method)