def _hardlink_atomic(self, src, dest, dir_info): head, tail = os.path.split(dest) hardlink_tmp = os.path.join(head, ".%s._mirrordist_hardlink_.%s" % \ (tail, os.getpid())) try: try: os.link(src, hardlink_tmp) except OSError as e: if e.errno != errno.EXDEV: msg = "hardlink %s from %s failed: %s" % \ (self.distfile, dir_info, e) self.scheduler.output(msg + '\n', background=True, log_path=self._log_path) logging.error(msg) return False try: os.rename(hardlink_tmp, dest) except OSError as e: msg = "hardlink rename '%s' from %s failed: %s" % \ (self.distfile, dir_info, e) self.scheduler.output(msg + '\n', background=True, log_path=self._log_path) logging.error(msg) return False finally: try: os.unlink(hardlink_tmp) except OSError: pass return True
def _lockfile_was_removed(lock_fd, lock_path):
"""
Check if lock_fd still refers to a file located at lock_path, since
the file may have been removed by a concurrent process that held the
lock earlier. This implementation includes support for NFS, where
stat is not reliable for removed files due to the default file
attribute cache behavior ('ac' mount option).
@param lock_fd: an open file descriptor for a lock file
@type lock_fd: int
@param lock_path: path of lock file
@type lock_path: str
@rtype: bool
@return: True if lock_path exists and corresponds to lock_fd, False otherwise
"""
try:
fstat_st = os.fstat(lock_fd)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
# Since stat is not reliable for removed files on NFS with the default
# file attribute cache behavior ('ac' mount option), create a temporary
# hardlink in order to prove that the file path exists on the NFS server.
hardlink_path = hardlock_name(lock_path)
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
try:
try:
os.link(lock_path, hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
hardlink_stat = os.stat(hardlink_path)
if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev:
return True
finally:
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return False
def _hardlink_atomic(self, src, dest, dir_info, symlink=False):
head, tail = os.path.split(dest)
hardlink_tmp = os.path.join(
head, ".%s._mirrordist_hardlink_.%s" % (tail, portage.getpid()))
try:
try:
if symlink:
os.symlink(src, hardlink_tmp)
else:
os.link(src, hardlink_tmp)
except OSError as e:
if e.errno != errno.EXDEV:
msg = "hardlink %s from %s failed: %s" % (
self.distfile,
dir_info,
e,
)
self.scheduler.output(msg + "\n",
background=True,
log_path=self._log_path)
logging.error(msg)
return False
try:
os.rename(hardlink_tmp, dest)
except OSError as e:
msg = "hardlink rename '%s' from %s failed: %s" % (
self.distfile,
dir_info,
e,
)
self.scheduler.output(msg + "\n",
background=True,
log_path=self._log_path)
logging.error(msg)
return False
finally:
try:
os.unlink(hardlink_tmp)
except OSError:
pass
return True
def hardlink_lockfile(lockfilename, max_wait=14400):
"""Does the NFS, hardlink shuffle to ensure locking on the disk.
We create a PRIVATE lockfile, that is just a placeholder on the disk.
Then we HARDLINK the real lockfile to that private file.
If our file can 2 references, then we have the lock. :)
Otherwise we lather, rise, and repeat.
We default to a 4 hour timeout.
"""
start_time = time.time()
myhardlock = hardlock_name(lockfilename)
reported_waiting = False
while(time.time() < (start_time + max_wait)):
# We only need it to exist.
myfd = os.open(myhardlock, os.O_CREAT|os.O_RDWR,0o660)
os.close(myfd)
if not os.path.exists(myhardlock):
raise FileNotFound(
_("Created lockfile is missing: %(filename)s") % \
{"filename" : myhardlock})
try:
res = os.link(myhardlock, lockfilename)
except OSError:
pass
if hardlink_is_mine(myhardlock, lockfilename):
# We have the lock.
if reported_waiting:
writemsg("\n", noiselevel=-1)
return True
if reported_waiting:
writemsg(".", noiselevel=-1)
else:
reported_waiting = True
from portage.const import PORTAGE_BIN_PATH
msg = _("\nWaiting on (hardlink) lockfile: (one '.' per 3 seconds)\n"
"%(bin_path)s/clean_locks can fix stuck locks.\n"
"Lockfile: %(lockfilename)s\n") % \
{"bin_path": PORTAGE_BIN_PATH, "lockfilename": lockfilename}
writemsg(msg, noiselevel=-1)
time.sleep(3)
os.unlink(myhardlock)
return False
def hardlink_lockfile(lockfilename, max_wait=14400):
"""Does the NFS, hardlink shuffle to ensure locking on the disk.
We create a PRIVATE lockfile, that is just a placeholder on the disk.
Then we HARDLINK the real lockfile to that private file.
If our file can 2 references, then we have the lock. :)
Otherwise we lather, rise, and repeat.
We default to a 4 hour timeout.
"""
start_time = time.time()
myhardlock = hardlock_name(lockfilename)
reported_waiting = False
while (time.time() < (start_time + max_wait)):
# We only need it to exist.
myfd = os.open(myhardlock, os.O_CREAT | os.O_RDWR, 0o660)
os.close(myfd)
if not os.path.exists(myhardlock):
raise FileNotFound(
_("Created lockfile is missing: %(filename)s") % \
{"filename" : myhardlock})
try:
res = os.link(myhardlock, lockfilename)
except OSError:
pass
if hardlink_is_mine(myhardlock, lockfilename):
# We have the lock.
if reported_waiting:
writemsg("\n", noiselevel=-1)
return True
if reported_waiting:
writemsg(".", noiselevel=-1)
else:
reported_waiting = True
from portage.const import PORTAGE_BIN_PATH
msg = _("\nWaiting on (hardlink) lockfile: (one '.' per 3 seconds)\n"
"%(bin_path)s/clean_locks can fix stuck locks.\n"
"Lockfile: %(lockfilename)s\n") % \
{"bin_path": PORTAGE_BIN_PATH, "lockfilename": lockfilename}
writemsg(msg, noiselevel=-1)
time.sleep(3)
os.unlink(myhardlock)
return False
def hardlink_lockfile(lockfilename, max_wait=DeprecationWarning,
waiting_msg=None, flags=0):
"""Does the NFS, hardlink shuffle to ensure locking on the disk.
We create a PRIVATE hardlink to the real lockfile, that is just a
placeholder on the disk.
If our file can 2 references, then we have the lock. :)
Otherwise we lather, rise, and repeat.
"""
if max_wait is not DeprecationWarning:
warnings.warn("The 'max_wait' parameter of "
"portage.locks.hardlink_lockfile() is now unused. Use "
"flags=os.O_NONBLOCK instead.",
DeprecationWarning, stacklevel=2)
global _quiet
out = None
displayed_waiting_msg = False
preexisting = os.path.exists(lockfilename)
myhardlock = hardlock_name(lockfilename)
# Since Python 3.4, chown requires int type (no proxies).
portage_gid = int(portage.data.portage_gid)
# myhardlock must not exist prior to our link() call, and we can
# safely unlink it since its file name is unique to our PID
try:
os.unlink(myhardlock)
except OSError as e:
if e.errno in (errno.ENOENT, errno.ESTALE):
pass
else:
func_call = "unlink('%s')" % myhardlock
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
else:
raise
while True:
# create lockfilename if it doesn't exist yet
try:
myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
except OSError as e:
func_call = "open('%s')" % lockfilename
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
elif e.errno == ReadOnlyFileSystem.errno:
raise ReadOnlyFileSystem(func_call)
else:
raise
else:
myfd_st = None
try:
myfd_st = os.fstat(myfd)
if not preexisting:
# Don't chown the file if it is preexisting, since we
# want to preserve existing permissions in that case.
if myfd_st.st_gid != portage_gid:
os.fchown(myfd, -1, portage_gid)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
writemsg("%s: fchown('%s', -1, %d)\n" % \
(e, lockfilename, portage_gid), noiselevel=-1)
writemsg(_("Cannot chown a lockfile: '%s'\n") % \
lockfilename, noiselevel=-1)
writemsg(_("Group IDs of current user: %s\n") % \
" ".join(str(n) for n in os.getgroups()),
noiselevel=-1)
else:
# another process has removed the file, so we'll have
# to create it again
continue
finally:
os.close(myfd)
# If fstat shows more than one hardlink, then it's extremely
# unlikely that the following link call will result in a lock,
# so optimize away the wasteful link call and sleep or raise
# TryAgain.
if myfd_st is not None and myfd_st.st_nlink < 2:
try:
os.link(lockfilename, myhardlock)
except OSError as e:
func_call = "link('%s', '%s')" % (lockfilename, myhardlock)
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
elif e.errno in (errno.ESTALE, errno.ENOENT):
# another process has removed the file, so we'll have
# to create it again
continue
else:
raise
else:
if hardlink_is_mine(myhardlock, lockfilename):
if out is not None:
out.eend(os.EX_OK)
break
try:
os.unlink(myhardlock)
except OSError as e:
# This should not happen, since the file name of
# myhardlock is unique to our host and PID,
# and the above link() call succeeded.
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
raise FileNotFound(myhardlock)
if flags & os.O_NONBLOCK:
raise TryAgain(lockfilename)
if out is None and not _quiet:
out = portage.output.EOutput()
if out is not None and not displayed_waiting_msg:
displayed_waiting_msg = True
if waiting_msg is None:
waiting_msg = _("waiting for lock on %s\n") % lockfilename
out.ebegin(waiting_msg)
time.sleep(_HARDLINK_POLL_LATENCY)
return True
def hardlink_lockfile(lockfilename,
max_wait=DeprecationWarning,
waiting_msg=None,
flags=0):
"""Does the NFS, hardlink shuffle to ensure locking on the disk.
We create a PRIVATE hardlink to the real lockfile, that is just a
placeholder on the disk.
If our file can 2 references, then we have the lock. :)
Otherwise we lather, rise, and repeat.
"""
if max_wait is not DeprecationWarning:
warnings.warn(
"The 'max_wait' parameter of "
"portage.locks.hardlink_lockfile() is now unused. Use "
"flags=os.O_NONBLOCK instead.",
DeprecationWarning,
stacklevel=2)
global _quiet
out = None
displayed_waiting_msg = False
preexisting = os.path.exists(lockfilename)
myhardlock = hardlock_name(lockfilename)
# Since Python 3.4, chown requires int type (no proxies).
portage_gid = int(portage.data.portage_gid)
# myhardlock must not exist prior to our link() call, and we can
# safely unlink it since its file name is unique to our PID
try:
os.unlink(myhardlock)
except OSError as e:
if e.errno in (errno.ENOENT, errno.ESTALE):
pass
else:
func_call = "unlink('%s')" % myhardlock
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
else:
raise
while True:
# create lockfilename if it doesn't exist yet
try:
myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660)
except OSError as e:
func_call = "open('%s')" % lockfilename
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
elif e.errno == ReadOnlyFileSystem.errno:
raise ReadOnlyFileSystem(func_call)
else:
raise
else:
myfd_st = None
try:
myfd_st = os.fstat(myfd)
if not preexisting:
# Don't chown the file if it is preexisting, since we
# want to preserve existing permissions in that case.
if portage.data.secpass >= 1 and myfd_st.st_gid != portage_gid:
os.fchown(myfd, -1, portage_gid)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
writemsg("%s: fchown('%s', -1, %d)\n" % \
(e, lockfilename, portage_gid), noiselevel=-1)
writemsg(_("Cannot chown a lockfile: '%s'\n") % \
lockfilename, noiselevel=-1)
writemsg(_("Group IDs of current user: %s\n") % \
" ".join(str(n) for n in os.getgroups()),
noiselevel=-1)
else:
# another process has removed the file, so we'll have
# to create it again
continue
finally:
os.close(myfd)
# If fstat shows more than one hardlink, then it's extremely
# unlikely that the following link call will result in a lock,
# so optimize away the wasteful link call and sleep or raise
# TryAgain.
if myfd_st is not None and myfd_st.st_nlink < 2:
try:
os.link(lockfilename, myhardlock)
except OSError as e:
func_call = "link('%s', '%s')" % (lockfilename, myhardlock)
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
elif e.errno in (errno.ESTALE, errno.ENOENT):
# another process has removed the file, so we'll have
# to create it again
continue
else:
raise
else:
if hardlink_is_mine(myhardlock, lockfilename):
if out is not None:
out.eend(os.EX_OK)
break
try:
os.unlink(myhardlock)
except OSError as e:
# This should not happen, since the file name of
# myhardlock is unique to our host and PID,
# and the above link() call succeeded.
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
raise FileNotFound(myhardlock)
if flags & os.O_NONBLOCK:
raise TryAgain(lockfilename)
if out is None and not _quiet:
out = portage.output.EOutput()
if out is not None and not displayed_waiting_msg:
displayed_waiting_msg = True
if waiting_msg is None:
waiting_msg = _("waiting for lock on %s\n") % lockfilename
out.ebegin(waiting_msg)
time.sleep(_HARDLINK_POLL_LATENCY)
return True
def _lockfile_was_removed(lock_fd, lock_path):
"""
Check if lock_fd still refers to a file located at lock_path, since
the file may have been removed by a concurrent process that held the
lock earlier. This implementation includes support for NFS, where
stat is not reliable for removed files due to the default file
attribute cache behavior ('ac' mount option).
@param lock_fd: an open file descriptor for a lock file
@type lock_fd: int
@param lock_path: path of lock file
@type lock_path: str
@rtype: bool
@return: True if lock_path exists and corresponds to lock_fd, False otherwise
"""
try:
fstat_st = os.fstat(lock_fd)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
# Since stat is not reliable for removed files on NFS with the default
# file attribute cache behavior ('ac' mount option), create a temporary
# hardlink in order to prove that the file path exists on the NFS server.
hardlink_path = hardlock_name(lock_path)
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
try:
try:
os.link(lock_path, hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
hardlink_stat = os.stat(hardlink_path)
if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev:
# Create another hardlink in order to detect whether or not
# hardlink inode numbers are expected to match. For example,
# inode numbers are not expected to match for sshfs.
inode_test = hardlink_path + '-inode-test'
try:
os.unlink(inode_test)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
try:
os.link(hardlink_path, inode_test)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
else:
if not os.path.samefile(hardlink_path, inode_test):
# This implies that inode numbers are not expected
# to match for this file system, so use a simple
# stat call to detect if lock_path has been removed.
return not os.path.exists(lock_path)
finally:
try:
os.unlink(inode_test)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
finally:
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return False
def test_gpkg_long_hardlink_path(self):
if sys.version_info.major < 3:
self.skipTest("Not support Python 2")
playground = ResolverPlayground(
user_config={
"make.conf": ('BINPKG_COMPRESS="none"',),
}
)
tmpdir = tempfile.mkdtemp()
try:
settings = playground.settings
path_name = (
"aaaabbbb/ccccdddd/eeeeffff/gggghhhh/iiiijjjj/kkkkllll/"
"mmmmnnnn/oooopppp/qqqqrrrr/sssstttt/uuuuvvvv/wwwwxxxx/"
)
file_name = (
"test-A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
"A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
"A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
)
orig_full_path = os.path.join(tmpdir, "orig", path_name)
os.makedirs(orig_full_path)
with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
test_file.write(urandom(1048576))
os.link(
os.path.join(orig_full_path, "test"),
os.path.join(orig_full_path, file_name),
)
gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
test_gpkg = gpkg(settings, "test", gpkg_file_loc)
check_result = test_gpkg._check_pre_image_files(
os.path.join(tmpdir, "orig")
)
self.assertEqual(check_result, (113, 158, 272, 1048576, 2097152))
test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
with open(gpkg_file_loc, "rb") as container:
# container
self.assertEqual(
test_gpkg._get_tar_format(container), tarfile.USTAR_FORMAT
)
with tarfile.open(gpkg_file_loc, "r") as container:
metadata = io.BytesIO(container.extractfile("test/metadata.tar").read())
self.assertEqual(
test_gpkg._get_tar_format(metadata), tarfile.USTAR_FORMAT
)
metadata.close()
image = io.BytesIO(container.extractfile("test/image.tar").read())
self.assertEqual(test_gpkg._get_tar_format(image), tarfile.GNU_FORMAT)
image.close()
test_gpkg.decompress(os.path.join(tmpdir, "test"))
r = compare_files(
os.path.join(tmpdir, "orig", path_name, file_name),
os.path.join(tmpdir, "test", path_name, file_name),
skipped_types=("atime", "mtime", "ctime"),
)
self.assertEqual(r, ())
finally:
shutil.rmtree(tmpdir)
def _lockfile_was_removed(lock_fd, lock_path):
"""
Check if lock_fd still refers to a file located at lock_path, since
the file may have been removed by a concurrent process that held the
lock earlier. This implementation includes support for NFS, where
stat is not reliable for removed files due to the default file
attribute cache behavior ('ac' mount option).
@param lock_fd: an open file descriptor for a lock file
@type lock_fd: int
@param lock_path: path of lock file
@type lock_path: str
@rtype: bool
@return: True if lock_path exists and corresponds to lock_fd, False otherwise
"""
try:
fstat_st = os.fstat(lock_fd)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
# Since stat is not reliable for removed files on NFS with the default
# file attribute cache behavior ('ac' mount option), create a temporary
# hardlink in order to prove that the file path exists on the NFS server.
hardlink_path = hardlock_name(lock_path)
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
try:
try:
os.link(lock_path, hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
hardlink_stat = os.stat(hardlink_path)
if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev:
# Create another hardlink in order to detect whether or not
# hardlink inode numbers are expected to match. For example,
# inode numbers are not expected to match for sshfs.
inode_test = hardlink_path + '-inode-test'
try:
os.unlink(inode_test)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
try:
os.link(hardlink_path, inode_test)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
else:
if not os.path.samefile(hardlink_path, inode_test):
# This implies that inode numbers are not expected
# to match for this file system, so use a simple
# stat call to detect if lock_path has been removed.
return not os.path.exists(lock_path)
finally:
try:
os.unlink(inode_test)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
finally:
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return False