Пример #1
0
	def _hardlink_atomic(self, src, dest, dir_info):

		head, tail = os.path.split(dest)
		hardlink_tmp = os.path.join(head, ".%s._mirrordist_hardlink_.%s" % \
			(tail, os.getpid()))

		try:
			try:
				os.link(src, hardlink_tmp)
			except OSError as e:
				if e.errno != errno.EXDEV:
					msg = "hardlink %s from %s failed: %s" % \
						(self.distfile, dir_info, e)
					self.scheduler.output(msg + '\n', background=True,
						log_path=self._log_path)
					logging.error(msg)
				return False

			try:
				os.rename(hardlink_tmp, dest)
			except OSError as e:
				msg = "hardlink rename '%s' from %s failed: %s" % \
					(self.distfile, dir_info, e)
				self.scheduler.output(msg + '\n', background=True,
					log_path=self._log_path)
				logging.error(msg)
				return False
		finally:
			try:
				os.unlink(hardlink_tmp)
			except OSError:
				pass

		return True
Пример #2
0
def _lockfile_was_removed(lock_fd, lock_path):
	"""
	Check if lock_fd still refers to a file located at lock_path, since
	the file may have been removed by a concurrent process that held the
	lock earlier. This implementation includes support for NFS, where
	stat is not reliable for removed files due to the default file
	attribute cache behavior ('ac' mount option).

	@param lock_fd: an open file descriptor for a lock file
	@type lock_fd: int
	@param lock_path: path of lock file
	@type lock_path: str
	@rtype: bool
	@return: True if lock_path exists and corresponds to lock_fd, False otherwise
	"""
	try:
		fstat_st = os.fstat(lock_fd)
	except OSError as e:
		if e.errno not in (errno.ENOENT, errno.ESTALE):
			_raise_exc(e)
		return True

	# Since stat is not reliable for removed files on NFS with the default
	# file attribute cache behavior ('ac' mount option), create a temporary
	# hardlink in order to prove that the file path exists on the NFS server.
	hardlink_path = hardlock_name(lock_path)
	try:
		os.unlink(hardlink_path)
	except OSError as e:
		if e.errno not in (errno.ENOENT, errno.ESTALE):
			_raise_exc(e)
	try:
		try:
			os.link(lock_path, hardlink_path)
		except OSError as e:
			if e.errno not in (errno.ENOENT, errno.ESTALE):
				_raise_exc(e)
			return True

		hardlink_stat = os.stat(hardlink_path)
		if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev:
			return True
	finally:
		try:
			os.unlink(hardlink_path)
		except OSError as e:
			if e.errno not in (errno.ENOENT, errno.ESTALE):
				_raise_exc(e)
	return False
Пример #3
0
    def _hardlink_atomic(self, src, dest, dir_info, symlink=False):

        head, tail = os.path.split(dest)
        hardlink_tmp = os.path.join(
            head, ".%s._mirrordist_hardlink_.%s" % (tail, portage.getpid()))

        try:
            try:
                if symlink:
                    os.symlink(src, hardlink_tmp)
                else:
                    os.link(src, hardlink_tmp)
            except OSError as e:
                if e.errno != errno.EXDEV:
                    msg = "hardlink %s from %s failed: %s" % (
                        self.distfile,
                        dir_info,
                        e,
                    )
                    self.scheduler.output(msg + "\n",
                                          background=True,
                                          log_path=self._log_path)
                    logging.error(msg)
                return False

            try:
                os.rename(hardlink_tmp, dest)
            except OSError as e:
                msg = "hardlink rename '%s' from %s failed: %s" % (
                    self.distfile,
                    dir_info,
                    e,
                )
                self.scheduler.output(msg + "\n",
                                      background=True,
                                      log_path=self._log_path)
                logging.error(msg)
                return False
        finally:
            try:
                os.unlink(hardlink_tmp)
            except OSError:
                pass

        return True
Пример #4
0
def hardlink_lockfile(lockfilename, max_wait=14400):
	"""Does the NFS, hardlink shuffle to ensure locking on the disk.
	We create a PRIVATE lockfile, that is just a placeholder on the disk.
	Then we HARDLINK the real lockfile to that private file.
	If our file can 2 references, then we have the lock. :)
	Otherwise we lather, rise, and repeat.
	We default to a 4 hour timeout.
	"""

	start_time = time.time()
	myhardlock = hardlock_name(lockfilename)
	reported_waiting = False
	
	while(time.time() < (start_time + max_wait)):
		# We only need it to exist.
		myfd = os.open(myhardlock, os.O_CREAT|os.O_RDWR,0o660)
		os.close(myfd)
	
		if not os.path.exists(myhardlock):
			raise FileNotFound(
				_("Created lockfile is missing: %(filename)s") % \
				{"filename" : myhardlock})

		try:
			res = os.link(myhardlock, lockfilename)
		except OSError:
			pass

		if hardlink_is_mine(myhardlock, lockfilename):
			# We have the lock.
			if reported_waiting:
				writemsg("\n", noiselevel=-1)
			return True

		if reported_waiting:
			writemsg(".", noiselevel=-1)
		else:
			reported_waiting = True
			from portage.const import PORTAGE_BIN_PATH
			msg = _("\nWaiting on (hardlink) lockfile: (one '.' per 3 seconds)\n"
				"%(bin_path)s/clean_locks can fix stuck locks.\n"
				"Lockfile: %(lockfilename)s\n") % \
				{"bin_path": PORTAGE_BIN_PATH, "lockfilename": lockfilename}
			writemsg(msg, noiselevel=-1)
		time.sleep(3)
	
	os.unlink(myhardlock)
	return False
Пример #5
0
def hardlink_lockfile(lockfilename, max_wait=14400):
    """Does the NFS, hardlink shuffle to ensure locking on the disk.
	We create a PRIVATE lockfile, that is just a placeholder on the disk.
	Then we HARDLINK the real lockfile to that private file.
	If our file can 2 references, then we have the lock. :)
	Otherwise we lather, rise, and repeat.
	We default to a 4 hour timeout.
	"""

    start_time = time.time()
    myhardlock = hardlock_name(lockfilename)
    reported_waiting = False

    while (time.time() < (start_time + max_wait)):
        # We only need it to exist.
        myfd = os.open(myhardlock, os.O_CREAT | os.O_RDWR, 0o660)
        os.close(myfd)

        if not os.path.exists(myhardlock):
            raise FileNotFound(
             _("Created lockfile is missing: %(filename)s") % \
             {"filename" : myhardlock})

        try:
            res = os.link(myhardlock, lockfilename)
        except OSError:
            pass

        if hardlink_is_mine(myhardlock, lockfilename):
            # We have the lock.
            if reported_waiting:
                writemsg("\n", noiselevel=-1)
            return True

        if reported_waiting:
            writemsg(".", noiselevel=-1)
        else:
            reported_waiting = True
            from portage.const import PORTAGE_BIN_PATH
            msg = _("\nWaiting on (hardlink) lockfile: (one '.' per 3 seconds)\n"
             "%(bin_path)s/clean_locks can fix stuck locks.\n"
             "Lockfile: %(lockfilename)s\n") % \
             {"bin_path": PORTAGE_BIN_PATH, "lockfilename": lockfilename}
            writemsg(msg, noiselevel=-1)
        time.sleep(3)

    os.unlink(myhardlock)
    return False
Пример #6
0
def hardlink_lockfile(lockfilename, max_wait=DeprecationWarning,
	waiting_msg=None, flags=0):
	"""Does the NFS, hardlink shuffle to ensure locking on the disk.
	We create a PRIVATE hardlink to the real lockfile, that is just a
	placeholder on the disk.
	If our file can 2 references, then we have the lock. :)
	Otherwise we lather, rise, and repeat.
	"""

	if max_wait is not DeprecationWarning:
		warnings.warn("The 'max_wait' parameter of "
			"portage.locks.hardlink_lockfile() is now unused. Use "
			"flags=os.O_NONBLOCK instead.",
			DeprecationWarning, stacklevel=2)

	global _quiet
	out = None
	displayed_waiting_msg = False
	preexisting = os.path.exists(lockfilename)
	myhardlock = hardlock_name(lockfilename)

	# Since Python 3.4, chown requires int type (no proxies).
	portage_gid = int(portage.data.portage_gid)

	# myhardlock must not exist prior to our link() call, and we can
	# safely unlink it since its file name is unique to our PID
	try:
		os.unlink(myhardlock)
	except OSError as e:
		if e.errno in (errno.ENOENT, errno.ESTALE):
			pass
		else:
			func_call = "unlink('%s')" % myhardlock
			if e.errno == OperationNotPermitted.errno:
				raise OperationNotPermitted(func_call)
			elif e.errno == PermissionDenied.errno:
				raise PermissionDenied(func_call)
			else:
				raise

	while True:
		# create lockfilename if it doesn't exist yet
		try:
			myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
		except OSError as e:
			func_call = "open('%s')" % lockfilename
			if e.errno == OperationNotPermitted.errno:
				raise OperationNotPermitted(func_call)
			elif e.errno == PermissionDenied.errno:
				raise PermissionDenied(func_call)
			elif e.errno == ReadOnlyFileSystem.errno:
				raise ReadOnlyFileSystem(func_call)
			else:
				raise
		else:
			myfd_st = None
			try:
				myfd_st = os.fstat(myfd)
				if not preexisting:
					# Don't chown the file if it is preexisting, since we
					# want to preserve existing permissions in that case.
					if myfd_st.st_gid != portage_gid:
						os.fchown(myfd, -1, portage_gid)
			except OSError as e:
				if e.errno not in (errno.ENOENT, errno.ESTALE):
					writemsg("%s: fchown('%s', -1, %d)\n" % \
						(e, lockfilename, portage_gid), noiselevel=-1)
					writemsg(_("Cannot chown a lockfile: '%s'\n") % \
						lockfilename, noiselevel=-1)
					writemsg(_("Group IDs of current user: %s\n") % \
						" ".join(str(n) for n in os.getgroups()),
						noiselevel=-1)
				else:
					# another process has removed the file, so we'll have
					# to create it again
					continue
			finally:
				os.close(myfd)

			# If fstat shows more than one hardlink, then it's extremely
			# unlikely that the following link call will result in a lock,
			# so optimize away the wasteful link call and sleep or raise
			# TryAgain.
			if myfd_st is not None and myfd_st.st_nlink < 2:
				try:
					os.link(lockfilename, myhardlock)
				except OSError as e:
					func_call = "link('%s', '%s')" % (lockfilename, myhardlock)
					if e.errno == OperationNotPermitted.errno:
						raise OperationNotPermitted(func_call)
					elif e.errno == PermissionDenied.errno:
						raise PermissionDenied(func_call)
					elif e.errno in (errno.ESTALE, errno.ENOENT):
						# another process has removed the file, so we'll have
						# to create it again
						continue
					else:
						raise
				else:
					if hardlink_is_mine(myhardlock, lockfilename):
						if out is not None:
							out.eend(os.EX_OK)
						break

					try:
						os.unlink(myhardlock)
					except OSError as e:
						# This should not happen, since the file name of
						# myhardlock is unique to our host and PID,
						# and the above link() call succeeded.
						if e.errno not in (errno.ENOENT, errno.ESTALE):
							raise
						raise FileNotFound(myhardlock)

		if flags & os.O_NONBLOCK:
			raise TryAgain(lockfilename)

		if out is None and not _quiet:
			out = portage.output.EOutput()
		if out is not None and not displayed_waiting_msg:
			displayed_waiting_msg = True
			if waiting_msg is None:
				waiting_msg = _("waiting for lock on %s\n") % lockfilename
			out.ebegin(waiting_msg)

		time.sleep(_HARDLINK_POLL_LATENCY)

	return True
Пример #7
0
def hardlink_lockfile(lockfilename,
                      max_wait=DeprecationWarning,
                      waiting_msg=None,
                      flags=0):
    """Does the NFS, hardlink shuffle to ensure locking on the disk.
	We create a PRIVATE hardlink to the real lockfile, that is just a
	placeholder on the disk.
	If our file can 2 references, then we have the lock. :)
	Otherwise we lather, rise, and repeat.
	"""

    if max_wait is not DeprecationWarning:
        warnings.warn(
            "The 'max_wait' parameter of "
            "portage.locks.hardlink_lockfile() is now unused. Use "
            "flags=os.O_NONBLOCK instead.",
            DeprecationWarning,
            stacklevel=2)

    global _quiet
    out = None
    displayed_waiting_msg = False
    preexisting = os.path.exists(lockfilename)
    myhardlock = hardlock_name(lockfilename)

    # Since Python 3.4, chown requires int type (no proxies).
    portage_gid = int(portage.data.portage_gid)

    # myhardlock must not exist prior to our link() call, and we can
    # safely unlink it since its file name is unique to our PID
    try:
        os.unlink(myhardlock)
    except OSError as e:
        if e.errno in (errno.ENOENT, errno.ESTALE):
            pass
        else:
            func_call = "unlink('%s')" % myhardlock
            if e.errno == OperationNotPermitted.errno:
                raise OperationNotPermitted(func_call)
            elif e.errno == PermissionDenied.errno:
                raise PermissionDenied(func_call)
            else:
                raise

    while True:
        # create lockfilename if it doesn't exist yet
        try:
            myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660)
        except OSError as e:
            func_call = "open('%s')" % lockfilename
            if e.errno == OperationNotPermitted.errno:
                raise OperationNotPermitted(func_call)
            elif e.errno == PermissionDenied.errno:
                raise PermissionDenied(func_call)
            elif e.errno == ReadOnlyFileSystem.errno:
                raise ReadOnlyFileSystem(func_call)
            else:
                raise
        else:
            myfd_st = None
            try:
                myfd_st = os.fstat(myfd)
                if not preexisting:
                    # Don't chown the file if it is preexisting, since we
                    # want to preserve existing permissions in that case.
                    if portage.data.secpass >= 1 and myfd_st.st_gid != portage_gid:
                        os.fchown(myfd, -1, portage_gid)
            except OSError as e:
                if e.errno not in (errno.ENOENT, errno.ESTALE):
                    writemsg("%s: fchown('%s', -1, %d)\n" % \
                     (e, lockfilename, portage_gid), noiselevel=-1)
                    writemsg(_("Cannot chown a lockfile: '%s'\n") % \
                     lockfilename, noiselevel=-1)
                    writemsg(_("Group IDs of current user: %s\n") % \
                     " ".join(str(n) for n in os.getgroups()),
                     noiselevel=-1)
                else:
                    # another process has removed the file, so we'll have
                    # to create it again
                    continue
            finally:
                os.close(myfd)

            # If fstat shows more than one hardlink, then it's extremely
            # unlikely that the following link call will result in a lock,
            # so optimize away the wasteful link call and sleep or raise
            # TryAgain.
            if myfd_st is not None and myfd_st.st_nlink < 2:
                try:
                    os.link(lockfilename, myhardlock)
                except OSError as e:
                    func_call = "link('%s', '%s')" % (lockfilename, myhardlock)
                    if e.errno == OperationNotPermitted.errno:
                        raise OperationNotPermitted(func_call)
                    elif e.errno == PermissionDenied.errno:
                        raise PermissionDenied(func_call)
                    elif e.errno in (errno.ESTALE, errno.ENOENT):
                        # another process has removed the file, so we'll have
                        # to create it again
                        continue
                    else:
                        raise
                else:
                    if hardlink_is_mine(myhardlock, lockfilename):
                        if out is not None:
                            out.eend(os.EX_OK)
                        break

                    try:
                        os.unlink(myhardlock)
                    except OSError as e:
                        # This should not happen, since the file name of
                        # myhardlock is unique to our host and PID,
                        # and the above link() call succeeded.
                        if e.errno not in (errno.ENOENT, errno.ESTALE):
                            raise
                        raise FileNotFound(myhardlock)

        if flags & os.O_NONBLOCK:
            raise TryAgain(lockfilename)

        if out is None and not _quiet:
            out = portage.output.EOutput()
        if out is not None and not displayed_waiting_msg:
            displayed_waiting_msg = True
            if waiting_msg is None:
                waiting_msg = _("waiting for lock on %s\n") % lockfilename
            out.ebegin(waiting_msg)

        time.sleep(_HARDLINK_POLL_LATENCY)

    return True
Пример #8
0
def _lockfile_was_removed(lock_fd, lock_path):
    """
	Check if lock_fd still refers to a file located at lock_path, since
	the file may have been removed by a concurrent process that held the
	lock earlier. This implementation includes support for NFS, where
	stat is not reliable for removed files due to the default file
	attribute cache behavior ('ac' mount option).

	@param lock_fd: an open file descriptor for a lock file
	@type lock_fd: int
	@param lock_path: path of lock file
	@type lock_path: str
	@rtype: bool
	@return: True if lock_path exists and corresponds to lock_fd, False otherwise
	"""
    try:
        fstat_st = os.fstat(lock_fd)
    except OSError as e:
        if e.errno not in (errno.ENOENT, errno.ESTALE):
            _raise_exc(e)
        return True

    # Since stat is not reliable for removed files on NFS with the default
    # file attribute cache behavior ('ac' mount option), create a temporary
    # hardlink in order to prove that the file path exists on the NFS server.
    hardlink_path = hardlock_name(lock_path)
    try:
        os.unlink(hardlink_path)
    except OSError as e:
        if e.errno not in (errno.ENOENT, errno.ESTALE):
            _raise_exc(e)
    try:
        try:
            os.link(lock_path, hardlink_path)
        except OSError as e:
            if e.errno not in (errno.ENOENT, errno.ESTALE):
                _raise_exc(e)
            return True

        hardlink_stat = os.stat(hardlink_path)
        if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev:
            # Create another hardlink in order to detect whether or not
            # hardlink inode numbers are expected to match. For example,
            # inode numbers are not expected to match for sshfs.
            inode_test = hardlink_path + '-inode-test'
            try:
                os.unlink(inode_test)
            except OSError as e:
                if e.errno not in (errno.ENOENT, errno.ESTALE):
                    _raise_exc(e)
            try:
                os.link(hardlink_path, inode_test)
            except OSError as e:
                if e.errno not in (errno.ENOENT, errno.ESTALE):
                    _raise_exc(e)
                return True
            else:
                if not os.path.samefile(hardlink_path, inode_test):
                    # This implies that inode numbers are not expected
                    # to match for this file system, so use a simple
                    # stat call to detect if lock_path has been removed.
                    return not os.path.exists(lock_path)
            finally:
                try:
                    os.unlink(inode_test)
                except OSError as e:
                    if e.errno not in (errno.ENOENT, errno.ESTALE):
                        _raise_exc(e)
            return True
    finally:
        try:
            os.unlink(hardlink_path)
        except OSError as e:
            if e.errno not in (errno.ENOENT, errno.ESTALE):
                _raise_exc(e)
    return False
Пример #9
0
    def test_gpkg_long_hardlink_path(self):
        if sys.version_info.major < 3:
            self.skipTest("Not support Python 2")

        playground = ResolverPlayground(
            user_config={
                "make.conf": ('BINPKG_COMPRESS="none"',),
            }
        )
        tmpdir = tempfile.mkdtemp()

        try:
            settings = playground.settings

            path_name = (
                "aaaabbbb/ccccdddd/eeeeffff/gggghhhh/iiiijjjj/kkkkllll/"
                "mmmmnnnn/oooopppp/qqqqrrrr/sssstttt/uuuuvvvv/wwwwxxxx/"
            )
            file_name = (
                "test-A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
                "A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
                "A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
            )
            orig_full_path = os.path.join(tmpdir, "orig", path_name)
            os.makedirs(orig_full_path)
            with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
                test_file.write(urandom(1048576))

            os.link(
                os.path.join(orig_full_path, "test"),
                os.path.join(orig_full_path, file_name),
            )

            gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
            test_gpkg = gpkg(settings, "test", gpkg_file_loc)

            check_result = test_gpkg._check_pre_image_files(
                os.path.join(tmpdir, "orig")
            )
            self.assertEqual(check_result, (113, 158, 272, 1048576, 2097152))

            test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
            with open(gpkg_file_loc, "rb") as container:
                # container
                self.assertEqual(
                    test_gpkg._get_tar_format(container), tarfile.USTAR_FORMAT
                )

            with tarfile.open(gpkg_file_loc, "r") as container:
                metadata = io.BytesIO(container.extractfile("test/metadata.tar").read())
                self.assertEqual(
                    test_gpkg._get_tar_format(metadata), tarfile.USTAR_FORMAT
                )
                metadata.close()

                image = io.BytesIO(container.extractfile("test/image.tar").read())
                self.assertEqual(test_gpkg._get_tar_format(image), tarfile.GNU_FORMAT)
                image.close()

            test_gpkg.decompress(os.path.join(tmpdir, "test"))
            r = compare_files(
                os.path.join(tmpdir, "orig", path_name, file_name),
                os.path.join(tmpdir, "test", path_name, file_name),
                skipped_types=("atime", "mtime", "ctime"),
            )
            self.assertEqual(r, ())
        finally:
            shutil.rmtree(tmpdir)
Пример #10
0
def _lockfile_was_removed(lock_fd, lock_path):
	"""
	Check if lock_fd still refers to a file located at lock_path, since
	the file may have been removed by a concurrent process that held the
	lock earlier. This implementation includes support for NFS, where
	stat is not reliable for removed files due to the default file
	attribute cache behavior ('ac' mount option).

	@param lock_fd: an open file descriptor for a lock file
	@type lock_fd: int
	@param lock_path: path of lock file
	@type lock_path: str
	@rtype: bool
	@return: True if lock_path exists and corresponds to lock_fd, False otherwise
	"""
	try:
		fstat_st = os.fstat(lock_fd)
	except OSError as e:
		if e.errno not in (errno.ENOENT, errno.ESTALE):
			_raise_exc(e)
		return True

	# Since stat is not reliable for removed files on NFS with the default
	# file attribute cache behavior ('ac' mount option), create a temporary
	# hardlink in order to prove that the file path exists on the NFS server.
	hardlink_path = hardlock_name(lock_path)
	try:
		os.unlink(hardlink_path)
	except OSError as e:
		if e.errno not in (errno.ENOENT, errno.ESTALE):
			_raise_exc(e)
	try:
		try:
			os.link(lock_path, hardlink_path)
		except OSError as e:
			if e.errno not in (errno.ENOENT, errno.ESTALE):
				_raise_exc(e)
			return True

		hardlink_stat = os.stat(hardlink_path)
		if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev:
			# Create another hardlink in order to detect whether or not
			# hardlink inode numbers are expected to match. For example,
			# inode numbers are not expected to match for sshfs.
			inode_test = hardlink_path + '-inode-test'
			try:
				os.unlink(inode_test)
			except OSError as e:
				if e.errno not in (errno.ENOENT, errno.ESTALE):
					_raise_exc(e)
			try:
				os.link(hardlink_path, inode_test)
			except OSError as e:
				if e.errno not in (errno.ENOENT, errno.ESTALE):
					_raise_exc(e)
				return True
			else:
				if not os.path.samefile(hardlink_path, inode_test):
					# This implies that inode numbers are not expected
					# to match for this file system, so use a simple
					# stat call to detect if lock_path has been removed.
					return not os.path.exists(lock_path)
			finally:
				try:
					os.unlink(inode_test)
				except OSError as e:
					if e.errno not in (errno.ENOENT, errno.ESTALE):
						_raise_exc(e)
			return True
	finally:
		try:
			os.unlink(hardlink_path)
		except OSError as e:
			if e.errno not in (errno.ENOENT, errno.ESTALE):
				_raise_exc(e)
	return False