def _lockfile_was_removed(lock_fd, lock_path):
"""
Check if lock_fd still refers to a file located at lock_path, since
the file may have been removed by a concurrent process that held the
lock earlier. This implementation includes support for NFS, where
stat is not reliable for removed files due to the default file
attribute cache behavior ('ac' mount option).
@param lock_fd: an open file descriptor for a lock file
@type lock_fd: int
@param lock_path: path of lock file
@type lock_path: str
@rtype: bool
@return: True if lock_path exists and corresponds to lock_fd, False otherwise
"""
try:
fstat_st = os.fstat(lock_fd)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
# Since stat is not reliable for removed files on NFS with the default
# file attribute cache behavior ('ac' mount option), create a temporary
# hardlink in order to prove that the file path exists on the NFS server.
hardlink_path = hardlock_name(lock_path)
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
try:
try:
os.link(lock_path, hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
hardlink_stat = os.stat(hardlink_path)
if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev:
return True
finally:
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return False
def prepare_build_dirs(myroot=None, settings=None, cleanup=False):
"""
The myroot parameter is ignored.
"""
myroot = None
if settings is None:
raise TypeError("settings argument is required")
mysettings = settings
clean_dirs = [mysettings["HOME"]]
# We enable cleanup when we want to make sure old cruft (such as the old
# environment) doesn't interfere with the current phase.
if cleanup and 'keeptemp' not in mysettings.features:
clean_dirs.append(mysettings["T"])
for clean_dir in clean_dirs:
try:
shutil.rmtree(clean_dir)
except OSError as oe:
if errno.ENOENT == oe.errno:
pass
elif errno.EPERM == oe.errno:
writemsg("%s\n" % oe, noiselevel=-1)
writemsg(_("Operation Not Permitted: rmtree('%s')\n") % \
clean_dir, noiselevel=-1)
return 1
else:
# Wrap with PermissionDenied if appropriate, so that callers
# display a short error message without a traceback.
_raise_exc(oe)
def makedirs(dir_path):
try:
os.makedirs(dir_path)
except OSError as oe:
if errno.EEXIST == oe.errno:
pass
elif errno.EPERM == oe.errno:
writemsg("%s\n" % oe, noiselevel=-1)
writemsg(_("Operation Not Permitted: makedirs('%s')\n") % \
dir_path, noiselevel=-1)
return False
else:
raise
return True
mysettings["PKG_LOGDIR"] = os.path.join(mysettings["T"], "logging")
mydirs = [os.path.dirname(mysettings["PORTAGE_BUILDDIR"])]
mydirs.append(os.path.dirname(mydirs[-1]))
try:
for mydir in mydirs:
ensure_dirs(mydir)
try:
apply_secpass_permissions(mydir,
gid=portage_gid, uid=portage_uid, mode=0o700, mask=0)
except PortageException:
if not os.path.isdir(mydir):
raise
for dir_key in ("PORTAGE_BUILDDIR", "HOME", "PKG_LOGDIR", "T"):
ensure_dirs(mysettings[dir_key], mode=0o755)
apply_secpass_permissions(mysettings[dir_key],
uid=portage_uid, gid=portage_gid)
except PermissionDenied as e:
writemsg(_("Permission Denied: %s\n") % str(e), noiselevel=-1)
return 1
except OperationNotPermitted as e:
writemsg(_("Operation Not Permitted: %s\n") % str(e), noiselevel=-1)
return 1
except FileNotFound as e:
writemsg(_("File Not Found: '%s'\n") % str(e), noiselevel=-1)
return 1
# Reset state for things like noauto and keepwork in FEATURES.
for x in ('.die_hooks',):
try:
os.unlink(os.path.join(mysettings['PORTAGE_BUILDDIR'], x))
except OSError:
pass
_prepare_workdir(mysettings)
if mysettings.get("EBUILD_PHASE") not in ("info", "fetch", "pretend"):
# Avoid spurious permissions adjustments when fetching with
# a temporary PORTAGE_TMPDIR setting (for fetchonly).
_prepare_features_dirs(mysettings)
def _lockfile_was_removed(lock_fd, lock_path):
"""
Check if lock_fd still refers to a file located at lock_path, since
the file may have been removed by a concurrent process that held the
lock earlier. This implementation includes support for NFS, where
stat is not reliable for removed files due to the default file
attribute cache behavior ('ac' mount option).
@param lock_fd: an open file descriptor for a lock file
@type lock_fd: int
@param lock_path: path of lock file
@type lock_path: str
@rtype: bool
@return: True if lock_path exists and corresponds to lock_fd, False otherwise
"""
try:
fstat_st = os.fstat(lock_fd)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
# Since stat is not reliable for removed files on NFS with the default
# file attribute cache behavior ('ac' mount option), create a temporary
# hardlink in order to prove that the file path exists on the NFS server.
hardlink_path = hardlock_name(lock_path)
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
try:
try:
os.link(lock_path, hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
hardlink_stat = os.stat(hardlink_path)
if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev:
# Create another hardlink in order to detect whether or not
# hardlink inode numbers are expected to match. For example,
# inode numbers are not expected to match for sshfs.
inode_test = hardlink_path + '-inode-test'
try:
os.unlink(inode_test)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
try:
os.link(hardlink_path, inode_test)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
else:
if not os.path.samefile(hardlink_path, inode_test):
# This implies that inode numbers are not expected
# to match for this file system, so use a simple
# stat call to detect if lock_path has been removed.
return not os.path.exists(lock_path)
finally:
try:
os.unlink(inode_test)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
finally:
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return False
def _lockfile_iteration(mypath,
wantnewlockfile=False,
unlinkfile=False,
waiting_msg=None,
flags=0):
"""
Acquire a lock on mypath, without retry. Return None if the lockfile
was removed by previous lock holder (caller must retry).
@param mypath: lock file path
@type mypath: str
@param wantnewlockfile: use a separate new lock file
@type wantnewlockfile: bool
@param unlinkfile: remove lock file prior to unlock
@type unlinkfile: bool
@param waiting_msg: message to show before blocking
@type waiting_msg: str
@param flags: lock flags (only supports os.O_NONBLOCK)
@type flags: int
@rtype: bool
@return: unlockfile tuple on success, None if retry is needed
"""
if not mypath:
raise InvalidData(_("Empty path given"))
# Since Python 3.4, chown requires int type (no proxies).
portage_gid = int(portage.data.portage_gid)
# Support for file object or integer file descriptor parameters is
# deprecated due to ambiguity in whether or not it's safe to close
# the file descriptor, making it prone to "Bad file descriptor" errors
# or file descriptor leaks.
if isinstance(mypath, basestring) and mypath[-1] == '/':
mypath = mypath[:-1]
lockfilename_path = mypath
if hasattr(mypath, 'fileno'):
warnings.warn(
"portage.locks.lockfile() support for "
"file object parameters is deprecated. Use a file path instead.",
DeprecationWarning,
stacklevel=2)
lockfilename_path = getattr(mypath, 'name', None)
mypath = mypath.fileno()
if isinstance(mypath, int):
warnings.warn(
"portage.locks.lockfile() support for integer file "
"descriptor parameters is deprecated. Use a file path instead.",
DeprecationWarning,
stacklevel=2)
lockfilename = mypath
wantnewlockfile = 0
unlinkfile = 0
elif wantnewlockfile:
base, tail = os.path.split(mypath)
lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
lockfilename_path = lockfilename
unlinkfile = 1
else:
lockfilename = mypath
if isinstance(mypath, basestring):
if not os.path.exists(os.path.dirname(mypath)):
raise DirectoryNotFound(os.path.dirname(mypath))
preexisting = os.path.exists(lockfilename)
old_mask = os.umask(000)
try:
while True:
try:
myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660)
except OSError as e:
if e.errno in (errno.ENOENT,
errno.ESTALE) and os.path.isdir(
os.path.dirname(lockfilename)):
# Retry required for NFS (see bug 636798).
continue
else:
_raise_exc(e)
else:
break
if not preexisting:
try:
if portage.data.secpass >= 1 and os.stat(
lockfilename).st_gid != portage_gid:
os.chown(lockfilename, -1, portage_gid)
except OSError as e:
if e.errno in (errno.ENOENT, errno.ESTALE):
os.close(myfd)
return None
else:
writemsg("%s: chown('%s', -1, %d)\n" % \
(e, lockfilename, portage_gid), noiselevel=-1)
writemsg(_("Cannot chown a lockfile: '%s'\n") % \
lockfilename, noiselevel=-1)
writemsg(_("Group IDs of current user: %s\n") % \
" ".join(str(n) for n in os.getgroups()),
noiselevel=-1)
finally:
os.umask(old_mask)
elif isinstance(mypath, int):
myfd = mypath
else:
raise ValueError(_("Unknown type passed in '%s': '%s'") % \
(type(mypath), mypath))
# try for a non-blocking lock, if it's held, throw a message
# we're waiting on lockfile and use a blocking attempt.
locking_method = portage._eintr_func_wrapper(_get_lock_fn())
try:
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
raise IOError(errno.ENOSYS, "Function not implemented")
locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError as e:
if not hasattr(e, "errno"):
raise
if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK):
# resource temp unavailable; eg, someone beat us to the lock.
if flags & os.O_NONBLOCK:
os.close(myfd)
raise TryAgain(mypath)
global _quiet
if _quiet:
out = None
else:
out = portage.output.EOutput()
if waiting_msg is None:
if isinstance(mypath, int):
waiting_msg = _("waiting for lock on fd %i") % myfd
else:
waiting_msg = _("waiting for lock on %s") % lockfilename
if out is not None:
out.ebegin(waiting_msg)
# try for the exclusive lock now.
enolock_msg_shown = False
while True:
try:
locking_method(myfd, fcntl.LOCK_EX)
except EnvironmentError as e:
if e.errno == errno.ENOLCK:
# This is known to occur on Solaris NFS (see
# bug #462694). Assume that the error is due
# to temporary exhaustion of record locks,
# and loop until one becomes available.
if not enolock_msg_shown:
enolock_msg_shown = True
if isinstance(mypath, int):
context_desc = _("Error while waiting "
"to lock fd %i") % myfd
else:
context_desc = _("Error while waiting "
"to lock '%s'") % lockfilename
writemsg("\n!!! %s: %s\n" % (context_desc, e),
noiselevel=-1)
time.sleep(_HARDLINK_POLL_LATENCY)
continue
if out is not None:
out.eend(1, str(e))
raise
else:
break
if out is not None:
out.eend(os.EX_OK)
elif e.errno in (errno.ENOSYS, ):
# We're not allowed to lock on this FS.
if not isinstance(lockfilename, int):
# If a file object was passed in, it's not safe
# to close the file descriptor because it may
# still be in use.
os.close(myfd)
lockfilename_path = _unicode_decode(lockfilename_path,
encoding=_encodings['fs'],
errors='strict')
if not isinstance(lockfilename_path, basestring):
raise
link_success = hardlink_lockfile(lockfilename_path,
waiting_msg=waiting_msg,
flags=flags)
if not link_success:
raise
lockfilename = lockfilename_path
locking_method = None
myfd = HARDLINK_FD
else:
raise
if isinstance(lockfilename,
basestring) and myfd != HARDLINK_FD and unlinkfile:
try:
removed = _lockfile_was_removed(myfd, lockfilename)
except Exception:
# Do not leak the file descriptor here.
os.close(myfd)
raise
else:
if removed:
# Removed by previous lock holder... Caller will retry...
os.close(myfd)
return None
if myfd != HARDLINK_FD:
# FD_CLOEXEC is enabled by default in Python >=3.4.
if sys.hexversion < 0x3040000:
try:
fcntl.FD_CLOEXEC
except AttributeError:
pass
else:
fcntl.fcntl(
myfd, fcntl.F_SETFD,
fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
_open_fds.add(myfd)
writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1)
return (lockfilename, myfd, unlinkfile, locking_method)
def _lockfile_was_removed(lock_fd, lock_path):
"""
Check if lock_fd still refers to a file located at lock_path, since
the file may have been removed by a concurrent process that held the
lock earlier. This implementation includes support for NFS, where
stat is not reliable for removed files due to the default file
attribute cache behavior ('ac' mount option).
@param lock_fd: an open file descriptor for a lock file
@type lock_fd: int
@param lock_path: path of lock file
@type lock_path: str
@rtype: bool
@return: True if lock_path exists and corresponds to lock_fd, False otherwise
"""
try:
fstat_st = os.fstat(lock_fd)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
# Since stat is not reliable for removed files on NFS with the default
# file attribute cache behavior ('ac' mount option), create a temporary
# hardlink in order to prove that the file path exists on the NFS server.
hardlink_path = hardlock_name(lock_path)
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
try:
try:
os.link(lock_path, hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
hardlink_stat = os.stat(hardlink_path)
if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev:
# Create another hardlink in order to detect whether or not
# hardlink inode numbers are expected to match. For example,
# inode numbers are not expected to match for sshfs.
inode_test = hardlink_path + '-inode-test'
try:
os.unlink(inode_test)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
try:
os.link(hardlink_path, inode_test)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
else:
if not os.path.samefile(hardlink_path, inode_test):
# This implies that inode numbers are not expected
# to match for this file system, so use a simple
# stat call to detect if lock_path has been removed.
return not os.path.exists(lock_path)
finally:
try:
os.unlink(inode_test)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
finally:
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return False
def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False,
waiting_msg=None, flags=0):
"""
Acquire a lock on mypath, without retry. Return None if the lockfile
was removed by previous lock holder (caller must retry).
@param mypath: lock file path
@type mypath: str
@param wantnewlockfile: use a separate new lock file
@type wantnewlockfile: bool
@param unlinkfile: remove lock file prior to unlock
@type unlinkfile: bool
@param waiting_msg: message to show before blocking
@type waiting_msg: str
@param flags: lock flags (only supports os.O_NONBLOCK)
@type flags: int
@rtype: bool
@return: unlockfile tuple on success, None if retry is needed
"""
if not mypath:
raise InvalidData(_("Empty path given"))
# Since Python 3.4, chown requires int type (no proxies).
portage_gid = int(portage.data.portage_gid)
# Support for file object or integer file descriptor parameters is
# deprecated due to ambiguity in whether or not it's safe to close
# the file descriptor, making it prone to "Bad file descriptor" errors
# or file descriptor leaks.
if isinstance(mypath, basestring) and mypath[-1] == '/':
mypath = mypath[:-1]
lockfilename_path = mypath
if hasattr(mypath, 'fileno'):
warnings.warn("portage.locks.lockfile() support for "
"file object parameters is deprecated. Use a file path instead.",
DeprecationWarning, stacklevel=2)
lockfilename_path = getattr(mypath, 'name', None)
mypath = mypath.fileno()
if isinstance(mypath, int):
warnings.warn("portage.locks.lockfile() support for integer file "
"descriptor parameters is deprecated. Use a file path instead.",
DeprecationWarning, stacklevel=2)
lockfilename = mypath
wantnewlockfile = 0
unlinkfile = 0
elif wantnewlockfile:
base, tail = os.path.split(mypath)
lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
lockfilename_path = lockfilename
unlinkfile = 1
else:
lockfilename = mypath
if isinstance(mypath, basestring):
if not os.path.exists(os.path.dirname(mypath)):
raise DirectoryNotFound(os.path.dirname(mypath))
preexisting = os.path.exists(lockfilename)
old_mask = os.umask(000)
try:
while True:
try:
myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
except OSError as e:
if e.errno in (errno.ENOENT, errno.ESTALE) and os.path.isdir(os.path.dirname(lockfilename)):
# Retry required for NFS (see bug 636798).
continue
else:
_raise_exc(e)
else:
break
if not preexisting:
try:
if portage.data.secpass >= 1 and os.stat(lockfilename).st_gid != portage_gid:
os.chown(lockfilename, -1, portage_gid)
except OSError as e:
if e.errno in (errno.ENOENT, errno.ESTALE):
return lockfile(mypath,
wantnewlockfile=wantnewlockfile,
unlinkfile=unlinkfile, waiting_msg=waiting_msg,
flags=flags)
else:
writemsg("%s: chown('%s', -1, %d)\n" % \
(e, lockfilename, portage_gid), noiselevel=-1)
writemsg(_("Cannot chown a lockfile: '%s'\n") % \
lockfilename, noiselevel=-1)
writemsg(_("Group IDs of current user: %s\n") % \
" ".join(str(n) for n in os.getgroups()),
noiselevel=-1)
finally:
os.umask(old_mask)
elif isinstance(mypath, int):
myfd = mypath
else:
raise ValueError(_("Unknown type passed in '%s': '%s'") % \
(type(mypath), mypath))
# try for a non-blocking lock, if it's held, throw a message
# we're waiting on lockfile and use a blocking attempt.
locking_method = portage._eintr_func_wrapper(_get_lock_fn())
try:
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
raise IOError(errno.ENOSYS, "Function not implemented")
locking_method(myfd, fcntl.LOCK_EX|fcntl.LOCK_NB)
except IOError as e:
if not hasattr(e, "errno"):
raise
if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK):
# resource temp unavailable; eg, someone beat us to the lock.
if flags & os.O_NONBLOCK:
os.close(myfd)
raise TryAgain(mypath)
global _quiet
if _quiet:
out = None
else:
out = portage.output.EOutput()
if waiting_msg is None:
if isinstance(mypath, int):
waiting_msg = _("waiting for lock on fd %i") % myfd
else:
waiting_msg = _("waiting for lock on %s") % lockfilename
if out is not None:
out.ebegin(waiting_msg)
# try for the exclusive lock now.
enolock_msg_shown = False
while True:
try:
locking_method(myfd, fcntl.LOCK_EX)
except EnvironmentError as e:
if e.errno == errno.ENOLCK:
# This is known to occur on Solaris NFS (see
# bug #462694). Assume that the error is due
# to temporary exhaustion of record locks,
# and loop until one becomes available.
if not enolock_msg_shown:
enolock_msg_shown = True
if isinstance(mypath, int):
context_desc = _("Error while waiting "
"to lock fd %i") % myfd
else:
context_desc = _("Error while waiting "
"to lock '%s'") % lockfilename
writemsg("\n!!! %s: %s\n" % (context_desc, e),
noiselevel=-1)
time.sleep(_HARDLINK_POLL_LATENCY)
continue
if out is not None:
out.eend(1, str(e))
raise
else:
break
if out is not None:
out.eend(os.EX_OK)
elif e.errno in (errno.ENOSYS,):
# We're not allowed to lock on this FS.
if not isinstance(lockfilename, int):
# If a file object was passed in, it's not safe
# to close the file descriptor because it may
# still be in use.
os.close(myfd)
lockfilename_path = _unicode_decode(lockfilename_path,
encoding=_encodings['fs'], errors='strict')
if not isinstance(lockfilename_path, basestring):
raise
link_success = hardlink_lockfile(lockfilename_path,
waiting_msg=waiting_msg, flags=flags)
if not link_success:
raise
lockfilename = lockfilename_path
locking_method = None
myfd = HARDLINK_FD
else:
raise
if isinstance(lockfilename, basestring) and myfd != HARDLINK_FD and unlinkfile:
try:
removed = _lockfile_was_removed(myfd, lockfilename)
except Exception:
# Do not leak the file descriptor here.
os.close(myfd)
raise
else:
if removed:
# Removed by previous lock holder... Caller will retry...
os.close(myfd)
return None
if myfd != HARDLINK_FD:
# FD_CLOEXEC is enabled by default in Python >=3.4.
if sys.hexversion < 0x3040000:
try:
fcntl.FD_CLOEXEC
except AttributeError:
pass
else:
fcntl.fcntl(myfd, fcntl.F_SETFD,
fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
_open_fds.add(myfd)
writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1)
return (lockfilename, myfd, unlinkfile, locking_method)
