Exemple #1
0
		def make_pipes():
			tempdir = tempfile.mkdtemp()
			fifo_path = os.path.join(tempdir, 'fifo')
			os.mkfifo(fifo_path)
			return ((os.open(fifo_path, os.O_NONBLOCK|os.O_RDONLY),
				os.open(fifo_path, os.O_NONBLOCK|os.O_WRONLY)),
				functools.partial(shutil.rmtree, tempdir))
	def _start_post_builddir_lock(self, lock_future=None, start_ipc_daemon=False):
		if lock_future is not None:
			if lock_future is not self._start_future:
				raise AssertionError('lock_future is not self._start_future')
			self._start_future = None
			if lock_future.cancelled():
				self._build_dir = None
				self.cancelled = True
				self._was_cancelled()
				self._async_wait()
				return

			lock_future.result()

		if start_ipc_daemon:
			self.settings['PORTAGE_IPC_DAEMON'] = "1"
			self._start_ipc_daemon()

		if self.fd_pipes is None:
			self.fd_pipes = {}
		null_fd = None
		if 0 not in self.fd_pipes and \
			self.phase not in self._phases_interactive_whitelist and \
			"interactive" not in self.settings.get("PROPERTIES", "").split():
			null_fd = os.open('/dev/null', os.O_RDONLY)
			self.fd_pipes[0] = null_fd

		try:
			SpawnProcess._start(self)
		finally:
			if null_fd is not None:
				os.close(null_fd)
Exemple #3
0
	def testLogfile(self):
		logfile = None
		try:
			fd, logfile = tempfile.mkstemp()
			os.close(fd)
			null_fd = os.open('/dev/null', os.O_RDWR)
			test_string = 2 * "blah blah blah\n"
			task_scheduler = TaskScheduler()
			proc = SpawnProcess(
				args=[BASH_BINARY, "-c",
				"echo -n '%s'" % test_string],
				env={}, fd_pipes={0:sys.stdin.fileno(), 1:null_fd, 2:null_fd},
				scheduler=task_scheduler.sched_iface,
				logfile=logfile)
			task_scheduler.add(proc)
			task_scheduler.run()
			os.close(null_fd)
			f = codecs.open(_unicode_encode(logfile,
				encoding=_encodings['fs'], errors='strict'),
				mode='r', encoding=_encodings['content'], errors='strict')
			log_content = f.read()
			f.close()
			# When logging passes through a pty, this comparison will fail
			# unless the oflag terminal attributes have the termios.OPOST
			# bit disabled. Otherwise, tranformations such as \n -> \r\n
			# may occur.
			self.assertEqual(test_string, log_content)
		finally:
			if logfile:
				try:
					os.unlink(logfile)
				except EnvironmentError as e:
					if e.errno != errno.ENOENT:
						raise
					del e
    def _start(self):
        self._files = self._files_dict()

        # File streams are in unbuffered mode since we do atomic
        # read and write of whole pickles.
        self._files.pipe_in = \
         os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK)

        # FD_CLOEXEC is enabled by default in Python >=3.4.
        if sys.hexversion < 0x3040000 and fcntl is not None:
            try:
                fcntl.FD_CLOEXEC
            except AttributeError:
                pass
            else:
                fcntl.fcntl(
                    self._files.pipe_in, fcntl.F_SETFD,
                    fcntl.fcntl(self._files.pipe_in, fcntl.F_GETFD)
                    | fcntl.FD_CLOEXEC)

        self._reg_id = self.scheduler.io_add_watch(self._files.pipe_in,
                                                   self._registered_events,
                                                   self._input_handler)

        self._registered = True
    def _reopen_input(self):
        """
		Re-open the input stream, in order to suppress
		POLLHUP events (bug #339976).
		"""
        self.scheduler.source_remove(self._reg_id)
        os.close(self._files.pipe_in)
        self._files.pipe_in = \
         os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK)

        # FD_CLOEXEC is enabled by default in Python >=3.4.
        if sys.hexversion < 0x3040000 and fcntl is not None:
            try:
                fcntl.FD_CLOEXEC
            except AttributeError:
                pass
            else:
                fcntl.fcntl(
                    self._files.pipe_in, fcntl.F_SETFD,
                    fcntl.fcntl(self._files.pipe_in, fcntl.F_GETFD)
                    | fcntl.FD_CLOEXEC)

        self._reg_id = self.scheduler.io_add_watch(self._files.pipe_in,
                                                   self._registered_events,
                                                   self._input_handler)
	def _start_post_builddir_lock(self, lock_future=None, start_ipc_daemon=False):
		if lock_future is not None:
			if lock_future is not self._start_future:
				raise AssertionError('lock_future is not self._start_future')
			self._start_future = None
			lock_future.result()

		if start_ipc_daemon:
			self.settings['PORTAGE_IPC_DAEMON'] = "1"
			self._start_ipc_daemon()

		if self.fd_pipes is None:
			self.fd_pipes = {}
		null_fd = None
		if 0 not in self.fd_pipes and \
			self.phase not in self._phases_interactive_whitelist and \
			"interactive" not in self.settings.get("PROPERTIES", "").split():
			null_fd = os.open('/dev/null', os.O_RDONLY)
			self.fd_pipes[0] = null_fd

		try:
			SpawnProcess._start(self)
		finally:
			if null_fd is not None:
				os.close(null_fd)
Exemple #7
0
	def testLogfile(self):
		logfile = None
		try:
			fd, logfile = tempfile.mkstemp()
			os.close(fd)
			null_fd = os.open('/dev/null', os.O_RDWR)
			test_string = 2 * "blah blah blah\n"
			scheduler = PollScheduler().sched_iface
			proc = SpawnProcess(
				args=[BASH_BINARY, "-c",
				"echo -n '%s'" % test_string],
				env={}, fd_pipes={0:sys.stdin.fileno(), 1:null_fd, 2:null_fd},
				scheduler=scheduler,
				logfile=logfile)
			proc.start()
			os.close(null_fd)
			self.assertEqual(proc.wait(), os.EX_OK)
			f = io.open(_unicode_encode(logfile,
				encoding=_encodings['fs'], errors='strict'),
				mode='r', encoding=_encodings['content'], errors='strict')
			log_content = f.read()
			f.close()
			# When logging passes through a pty, this comparison will fail
			# unless the oflag terminal attributes have the termios.OPOST
			# bit disabled. Otherwise, tranformations such as \n -> \r\n
			# may occur.
			self.assertEqual(test_string, log_content)
		finally:
			if logfile:
				try:
					os.unlink(logfile)
				except EnvironmentError as e:
					if e.errno != errno.ENOENT:
						raise
					del e
Exemple #8
0
    def _start_post_builddir_lock(self, lock_future=None, start_ipc_daemon=False):
        if lock_future is not None:
            if lock_future is not self._start_future:
                raise AssertionError("lock_future is not self._start_future")
            self._start_future = None
            if lock_future.cancelled():
                self._build_dir = None
                self.cancelled = True
                self._was_cancelled()
                self._async_wait()
                return

            lock_future.result()

        if start_ipc_daemon:
            self.settings["PORTAGE_IPC_DAEMON"] = "1"
            self._start_ipc_daemon()

        if self.fd_pipes is None:
            self.fd_pipes = {}
        null_fd = None
        if (
            0 not in self.fd_pipes
            and self.phase not in self._phases_interactive_whitelist
            and "interactive" not in self.settings.get("PROPERTIES", "").split()
        ):
            null_fd = os.open("/dev/null", os.O_RDONLY)
            self.fd_pipes[0] = null_fd

        self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD")
        try:
            SpawnProcess._start(self)
        finally:
            if null_fd is not None:
                os.close(null_fd)
Exemple #9
0
    def _fetch_uri(self, uri):

        if self.config.options.dry_run:
            # Simply report success.
            logging.info("dry-run: fetch '%s' from '%s'" %
                         (self.distfile, uri))
            self._success()
            self.returncode = os.EX_OK
            self._async_wait()
            return

        if self.config.options.temp_dir:
            self._fetch_tmp_dir_info = "temp-dir"
            distdir = self.config.options.temp_dir
        else:
            self._fetch_tmp_dir_info = "distfiles"
            distdir = self.config.options.distfiles

        tmp_basename = self.distfile + "._emirrordist_fetch_.%s" % portage.getpid(
        )

        variables = {"DISTDIR": distdir, "URI": uri, "FILE": tmp_basename}

        self._fetch_tmp_file = os.path.join(distdir, tmp_basename)

        try:
            os.unlink(self._fetch_tmp_file)
        except OSError:
            pass

        args = portage.util.shlex_split(default_fetchcommand)
        args = [portage.util.varexpand(x, mydict=variables) for x in args]

        args = [
            _unicode_encode(x, encoding=_encodings["fs"], errors="strict")
            for x in args
        ]

        null_fd = os.open(os.devnull, os.O_RDONLY)
        fetcher = PopenProcess(
            background=self.background,
            proc=subprocess.Popen(args,
                                  stdin=null_fd,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT),
            scheduler=self.scheduler,
        )
        os.close(null_fd)

        fetcher.pipe_reader = PipeLogger(
            background=self.background,
            input_fd=fetcher.proc.stdout,
            log_file_path=self._log_path,
            scheduler=self.scheduler,
        )

        self._start_task(fetcher, self._fetcher_exit)
	def _check_sig_key(self):
		null_fd = os.open('/dev/null', os.O_RDONLY)
		popen_proc = PopenProcess(proc=subprocess.Popen(
			["gpg", "--verify", self._manifest_path],
			stdin=null_fd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT),
			pipe_reader=PipeReader())
		os.close(null_fd)
		popen_proc.pipe_reader.input_files = {
			"producer" : popen_proc.proc.stdout}
		self._start_task(popen_proc, self._check_sig_key_exit)
Exemple #11
0
    def _start(self):
        self._files = self._files_dict()
        input_fd = os.open(self.input_fifo, os.O_RDONLY | os.O_NONBLOCK)
        self._files.pipe_in = os.fdopen(input_fd, 'rb')

        self._reg_id = self.scheduler.register(self._files.pipe_in.fileno(),
                                               self._registered_events,
                                               self._input_handler)

        self._registered = True
	def _check_sig_key(self):
		null_fd = os.open('/dev/null', os.O_RDONLY)
		popen_proc = PopenProcess(proc=subprocess.Popen(
			["gpg", "--verify", self._manifest_path],
			stdin=null_fd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT),
			pipe_reader=PipeReader())
		os.close(null_fd)
		popen_proc.pipe_reader.input_files = {
			"producer" : popen_proc.proc.stdout}
		self._start_task(popen_proc, self._check_sig_key_exit)
Exemple #13
0
    def _reopen_input(self):
        """
        Re-open the input stream, in order to suppress
        POLLHUP events (bug #339976).
        """
        self.scheduler.remove_reader(self._files.pipe_in)
        os.close(self._files.pipe_in)
        self._files.pipe_in = os.open(self.input_fifo, os.O_RDONLY | os.O_NONBLOCK)

        self.scheduler.add_reader(self._files.pipe_in, self._input_handler)
	def _start(self):
		self._files = self._files_dict()
		input_fd = os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK)
		self._files.pipe_in = os.fdopen(input_fd, 'rb')

		self._reg_id = self.scheduler.register(
			self._files.pipe_in.fileno(),
			self._registered_events, self._input_handler)

		self._registered = True
Exemple #15
0
    def _start(self):
        self._files = self._files_dict()

        # File streams are in unbuffered mode since we do atomic
        # read and write of whole pickles.
        self._files.pipe_in = os.open(self.input_fifo, os.O_RDONLY | os.O_NONBLOCK)

        self.scheduler.add_reader(self._files.pipe_in, self._input_handler)

        self._registered = True
Exemple #16
0
    def _reopen_input(self):
        """
		Re-open the input stream, in order to suppress
		POLLHUP events (bug #339976).
		"""
        self.scheduler.source_remove(self._reg_id)
        os.close(self._files.pipe_in)
        self._files.pipe_in = \
         os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK)
        self._reg_id = self.scheduler.io_add_watch(self._files.pipe_in,
                                                   self._registered_events,
                                                   self._input_handler)
Exemple #17
0
	def _reopen_input(self):
		"""
		Re-open the input stream, in order to suppress
		POLLHUP events (bug #339976).
		"""
		self.scheduler.unregister(self._reg_id)
		os.close(self._files.pipe_in)
		self._files.pipe_in = \
			os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK)
		self._reg_id = self.scheduler.register(
			self._files.pipe_in,
			self._registered_events, self._input_handler)
Exemple #18
0
	def _start(self):
		self._files = self._files_dict()

		# File streams are in unbuffered mode since we do atomic
		# read and write of whole pickles.
		self._files.pipe_in = \
			os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK)

		self._reg_id = self.scheduler.register(
			self._files.pipe_in,
			self._registered_events, self._input_handler)

		self._registered = True
Exemple #19
0
 def testLogfile(self):
     from portage import settings, spawn
     from tempfile import mkstemp
     logfile = None
     try:
         fd, logfile = mkstemp()
         os.close(fd)
         null_fd = os.open('/dev/null', os.O_RDWR)
         test_string = 2 * "blah blah blah\n"
         # Test cases are unique because they run inside src_test() which
         # may or may not already be running within a sandbox. Interaction
         # with SANDBOX_* variables may trigger unwanted sandbox violations
         # that are only reproducible with certain combinations of sandbox,
         # usersandbox, and userpriv FEATURES. Attempts to filter SANDBOX_*
         # variables can interfere with a currently running sandbox
         # instance. Therefore, use free=1 here to avoid potential
         # interactions (see bug #190268).
         spawn("echo -n '%s'" % test_string,
               settings,
               logfile=logfile,
               free=1,
               fd_pipes={
                   0: sys.stdin.fileno(),
                   1: null_fd,
                   2: null_fd
               })
         os.close(null_fd)
         f = codecs.open(_unicode_encode(logfile,
                                         encoding=_encodings['fs'],
                                         errors='strict'),
                         mode='r',
                         encoding=_encodings['content'],
                         errors='strict')
         log_content = f.read()
         f.close()
         # When logging passes through a pty, this comparison will fail
         # unless the oflag terminal attributes have the termios.OPOST
         # bit disabled. Otherwise, tranformations such as \n -> \r\n
         # may occur.
         self.assertEqual(test_string, log_content)
     finally:
         if logfile:
             try:
                 os.unlink(logfile)
             except EnvironmentError as e:
                 if e.errno != errno.ENOENT:
                     raise
                 del e
Exemple #20
0
def hardlink_lockfile(lockfilename, max_wait=14400):
    """Does the NFS, hardlink shuffle to ensure locking on the disk.
	We create a PRIVATE lockfile, that is just a placeholder on the disk.
	Then we HARDLINK the real lockfile to that private file.
	If our file can 2 references, then we have the lock. :)
	Otherwise we lather, rise, and repeat.
	We default to a 4 hour timeout.
	"""

    start_time = time.time()
    myhardlock = hardlock_name(lockfilename)
    reported_waiting = False

    while (time.time() < (start_time + max_wait)):
        # We only need it to exist.
        myfd = os.open(myhardlock, os.O_CREAT | os.O_RDWR, 0o660)
        os.close(myfd)

        if not os.path.exists(myhardlock):
            raise FileNotFound(
             _("Created lockfile is missing: %(filename)s") % \
             {"filename" : myhardlock})

        try:
            res = os.link(myhardlock, lockfilename)
        except OSError:
            pass

        if hardlink_is_mine(myhardlock, lockfilename):
            # We have the lock.
            if reported_waiting:
                writemsg("\n", noiselevel=-1)
            return True

        if reported_waiting:
            writemsg(".", noiselevel=-1)
        else:
            reported_waiting = True
            from portage.const import PORTAGE_BIN_PATH
            msg = _("\nWaiting on (hardlink) lockfile: (one '.' per 3 seconds)\n"
             "%(bin_path)s/clean_locks can fix stuck locks.\n"
             "Lockfile: %(lockfilename)s\n") % \
             {"bin_path": PORTAGE_BIN_PATH, "lockfilename": lockfilename}
            writemsg(msg, noiselevel=-1)
        time.sleep(3)

    os.unlink(myhardlock)
    return False
def hardlink_lockfile(lockfilename, max_wait=14400):
	"""Does the NFS, hardlink shuffle to ensure locking on the disk.
	We create a PRIVATE lockfile, that is just a placeholder on the disk.
	Then we HARDLINK the real lockfile to that private file.
	If our file can 2 references, then we have the lock. :)
	Otherwise we lather, rise, and repeat.
	We default to a 4 hour timeout.
	"""

	start_time = time.time()
	myhardlock = hardlock_name(lockfilename)
	reported_waiting = False
	
	while(time.time() < (start_time + max_wait)):
		# We only need it to exist.
		myfd = os.open(myhardlock, os.O_CREAT|os.O_RDWR,0o660)
		os.close(myfd)
	
		if not os.path.exists(myhardlock):
			raise FileNotFound(
				_("Created lockfile is missing: %(filename)s") % \
				{"filename" : myhardlock})

		try:
			res = os.link(myhardlock, lockfilename)
		except OSError:
			pass

		if hardlink_is_mine(myhardlock, lockfilename):
			# We have the lock.
			if reported_waiting:
				writemsg("\n", noiselevel=-1)
			return True

		if reported_waiting:
			writemsg(".", noiselevel=-1)
		else:
			reported_waiting = True
			from portage.const import PORTAGE_BIN_PATH
			msg = _("\nWaiting on (hardlink) lockfile: (one '.' per 3 seconds)\n"
				"%(bin_path)s/clean_locks can fix stuck locks.\n"
				"Lockfile: %(lockfilename)s\n") % \
				{"bin_path": PORTAGE_BIN_PATH, "lockfilename": lockfilename}
			writemsg(msg, noiselevel=-1)
		time.sleep(3)
	
	os.unlink(myhardlock)
	return False
Exemple #22
0
   def __init__(self, environment):
      self.conf = environment
      pidfile = open(self.conf.get('pid_file'), 'w')

      if (self.conf.get('no_daemon') == False):
         print "forking to background"
         if (os.fork() == 0):
             os.setpgid(0,0);
             pidfile.write(str(os.getpid()));
             pidfile.close();
             fd = os.open("/dev/null", os.O_WRONLY);
             os.dup2(fd,1);
             os.close(fd);
             #self.main_loop()
         else:
            sys.exit()

      else:
           print "Keeping in foreground"
           pidfile.write(str(os.getpid()));
           pidfile.close();
 def _send_reply(self, reply):
     # File streams are in unbuffered mode since we do atomic
     # read and write of whole pickles. Use non-blocking mode so
     # we don't hang if the client is killed before we can send
     # the reply. We rely on the client opening the other side
     # of this fifo before it sends its request, since otherwise
     # we'd have a race condition with this open call raising
     # ENXIO if the client hasn't opened the fifo yet.
     try:
         output_fd = os.open(self.output_fifo, os.O_WRONLY | os.O_NONBLOCK)
         try:
             os.write(output_fd, pickle.dumps(reply))
         finally:
             os.close(output_fd)
     except OSError as e:
         # This probably means that the client has been killed,
         # which causes open to fail with ENXIO.
         writemsg_level(
          "!!! EbuildIpcDaemon %s: %s\n" % \
          (_('failed to send reply'), e),
          level=logging.ERROR, noiselevel=-1)
Exemple #24
0
    def _run(self):

        syncfs_failed = False
        syncfs = self._get_syncfs()

        if syncfs is not None:
            for path in self.paths:
                try:
                    fd = os.open(path, os.O_RDONLY)
                except OSError:
                    pass
                else:
                    try:
                        if syncfs(fd) != 0:
                            # Happens with PyPy (bug #446610)
                            syncfs_failed = True
                    finally:
                        os.close(fd)

        if syncfs is None or syncfs_failed:
            return 1
        return os.EX_OK
Exemple #25
0
 def testLogfile(self):
     logfile = None
     try:
         fd, logfile = tempfile.mkstemp()
         os.close(fd)
         null_fd = os.open("/dev/null", os.O_RDWR)
         test_string = 2 * "blah blah blah\n"
         scheduler = PollScheduler().sched_iface
         proc = SpawnProcess(
             args=[BASH_BINARY, "-c", "echo -n '%s'" % test_string],
             env={},
             fd_pipes={0: sys.stdin.fileno(), 1: null_fd, 2: null_fd},
             scheduler=scheduler,
             logfile=logfile,
         )
         proc.start()
         os.close(null_fd)
         self.assertEqual(proc.wait(), os.EX_OK)
         f = io.open(
             _unicode_encode(logfile, encoding=_encodings["fs"], errors="strict"),
             mode="r",
             encoding=_encodings["content"],
             errors="strict",
         )
         log_content = f.read()
         f.close()
         # When logging passes through a pty, this comparison will fail
         # unless the oflag terminal attributes have the termios.OPOST
         # bit disabled. Otherwise, tranformations such as \n -> \r\n
         # may occur.
         self.assertEqual(test_string, log_content)
     finally:
         if logfile:
             try:
                 os.unlink(logfile)
             except EnvironmentError as e:
                 if e.errno != errno.ENOENT:
                     raise
                 del e
Exemple #26
0
	def _run(self):

		syncfs_failed = False
		syncfs = self._get_syncfs()

		if syncfs is not None:
			for path in self.paths:
				try:
					fd = os.open(path, os.O_RDONLY)
				except OSError:
					pass
				else:
					try:
						if syncfs(fd) != 0:
							# Happens with PyPy (bug #446610)
							syncfs_failed = True
					finally:
						os.close(fd)

		if syncfs is None or syncfs_failed:
			return 1
		return os.EX_OK
Exemple #27
0
	def _send_reply(self, reply):
		# File streams are in unbuffered mode since we do atomic
		# read and write of whole pickles. Use non-blocking mode so
		# we don't hang if the client is killed before we can send
		# the reply. We rely on the client opening the other side
		# of this fifo before it sends its request, since otherwise
		# we'd have a race condition with this open call raising
		# ENXIO if the client hasn't opened the fifo yet.
		try:
			output_fd = os.open(self.output_fifo,
				os.O_WRONLY | os.O_NONBLOCK)
			try:
				os.write(output_fd, pickle.dumps(reply))
			finally:
				os.close(output_fd)
		except OSError as e:
			# This probably means that the client has been killed,
			# which causes open to fail with ENXIO.
			writemsg_level(
				"!!! EbuildIpcDaemon %s: %s\n" % \
				(_('failed to send reply'), e),
				level=logging.ERROR, noiselevel=-1)
	def testLogfile(self):
		from portage import settings, spawn
		from tempfile import mkstemp
		logfile = None
		try:
			fd, logfile = mkstemp()
			os.close(fd)
			null_fd = os.open('/dev/null', os.O_RDWR)
			test_string = 2 * "blah blah blah\n"
			# Test cases are unique because they run inside src_test() which
			# may or may not already be running within a sandbox. Interaction
			# with SANDBOX_* variables may trigger unwanted sandbox violations
			# that are only reproducible with certain combinations of sandbox,
			# usersandbox, and userpriv FEATURES. Attempts to filter SANDBOX_*
			# variables can interfere with a currently running sandbox
			# instance. Therefore, use free=1 here to avoid potential
			# interactions (see bug #190268).
			spawn("echo -n '%s'" % test_string, settings, logfile=logfile,
				free=1, fd_pipes={0:sys.stdin.fileno(), 1:null_fd, 2:null_fd})
			os.close(null_fd)
			f = codecs.open(_unicode_encode(logfile,
				encoding=_encodings['fs'], errors='strict'),
				mode='r', encoding=_encodings['content'], errors='strict')
			log_content = f.read()
			f.close()
			# When logging passes through a pty, this comparison will fail
			# unless the oflag terminal attributes have the termios.OPOST
			# bit disabled. Otherwise, tranformations such as \n -> \r\n
			# may occur.
			self.assertEqual(test_string, log_content)
		finally:
			if logfile:
				try:
					os.unlink(logfile)
				except EnvironmentError as e:
					if e.errno != errno.ENOENT:
						raise
					del e
Exemple #29
0
	def _start(self):
		self._files = self._files_dict()

		# File streams are in unbuffered mode since we do atomic
		# read and write of whole pickles.
		self._files.pipe_in = \
			os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK)

		# FD_CLOEXEC is enabled by default in Python >=3.4.
		if sys.hexversion < 0x3040000 and fcntl is not None:
			try:
				fcntl.FD_CLOEXEC
			except AttributeError:
				pass
			else:
				fcntl.fcntl(self._files.pipe_in, fcntl.F_SETFD,
					fcntl.fcntl(self._files.pipe_in,
						fcntl.F_GETFD) | fcntl.FD_CLOEXEC)

		self.scheduler.add_reader(
			self._files.pipe_in,
			self._input_handler)

		self._registered = True
Exemple #30
0
	def _reopen_input(self):
		"""
		Re-open the input stream, in order to suppress
		POLLHUP events (bug #339976).
		"""
		self.scheduler.remove_reader(self._files.pipe_in)
		os.close(self._files.pipe_in)
		self._files.pipe_in = \
			os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK)

		# FD_CLOEXEC is enabled by default in Python >=3.4.
		if sys.hexversion < 0x3040000 and fcntl is not None:
			try:
				fcntl.FD_CLOEXEC
			except AttributeError:
				pass
			else:
				fcntl.fcntl(self._files.pipe_in, fcntl.F_SETFD,
					fcntl.fcntl(self._files.pipe_in,
						fcntl.F_GETFD) | fcntl.FD_CLOEXEC)

		self.scheduler.add_reader(
			self._files.pipe_in,
			self._input_handler)
Exemple #31
0
	def _fetch_uri(self, uri):

		if self.config.options.dry_run:
			# Simply report success.
			logging.info("dry-run: fetch '%s' from '%s'" %
				(self.distfile, uri))
			self._success()
			self.returncode = os.EX_OK
			self.wait()
			return

		if self.config.options.temp_dir:
			self._fetch_tmp_dir_info = 'temp-dir'
			distdir = self.config.options.temp_dir
		else:
			self._fetch_tmp_dir_info = 'distfiles'
			distdir = self.config.options.distfiles

		tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % os.getpid()

		variables = {
			"DISTDIR": distdir,
			"URI":     uri,
			"FILE":    tmp_basename
		}

		self._fetch_tmp_file = os.path.join(distdir, tmp_basename)

		try:
			os.unlink(self._fetch_tmp_file)
		except OSError:
			pass

		args = portage.util.shlex_split(default_fetchcommand)
		args = [portage.util.varexpand(x, mydict=variables)
			for x in args]

		if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
			not os.path.isabs(args[0]):
			# Python 3.1 _execvp throws TypeError for non-absolute executable
			# path passed as bytes (see https://bugs.python.org/issue8513).
			fullname = portage.process.find_binary(args[0])
			if fullname is None:
				raise portage.exception.CommandNotFound(args[0])
			args[0] = fullname

		args = [_unicode_encode(x,
			encoding=_encodings['fs'], errors='strict') for x in args]

		null_fd = os.open(os.devnull, os.O_RDONLY)
		fetcher = PopenProcess(background=self.background,
			proc=subprocess.Popen(args, stdin=null_fd,
			stdout=subprocess.PIPE, stderr=subprocess.STDOUT),
			scheduler=self.scheduler)
		os.close(null_fd)

		fetcher.pipe_reader = PipeLogger(background=self.background,
			input_fd=fetcher.proc.stdout, log_file_path=self._log_path,
			scheduler=self.scheduler)

		self._start_task(fetcher, self._fetcher_exit)
Exemple #32
0
def hardlink_lockfile(lockfilename,
                      max_wait=DeprecationWarning,
                      waiting_msg=None,
                      flags=0):
    """Does the NFS, hardlink shuffle to ensure locking on the disk.
	We create a PRIVATE hardlink to the real lockfile, that is just a
	placeholder on the disk.
	If our file can 2 references, then we have the lock. :)
	Otherwise we lather, rise, and repeat.
	"""

    if max_wait is not DeprecationWarning:
        warnings.warn(
            "The 'max_wait' parameter of "
            "portage.locks.hardlink_lockfile() is now unused. Use "
            "flags=os.O_NONBLOCK instead.",
            DeprecationWarning,
            stacklevel=2)

    global _quiet
    out = None
    displayed_waiting_msg = False
    preexisting = os.path.exists(lockfilename)
    myhardlock = hardlock_name(lockfilename)

    # Since Python 3.4, chown requires int type (no proxies).
    portage_gid = int(portage.data.portage_gid)

    # myhardlock must not exist prior to our link() call, and we can
    # safely unlink it since its file name is unique to our PID
    try:
        os.unlink(myhardlock)
    except OSError as e:
        if e.errno in (errno.ENOENT, errno.ESTALE):
            pass
        else:
            func_call = "unlink('%s')" % myhardlock
            if e.errno == OperationNotPermitted.errno:
                raise OperationNotPermitted(func_call)
            elif e.errno == PermissionDenied.errno:
                raise PermissionDenied(func_call)
            else:
                raise

    while True:
        # create lockfilename if it doesn't exist yet
        try:
            myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660)
        except OSError as e:
            func_call = "open('%s')" % lockfilename
            if e.errno == OperationNotPermitted.errno:
                raise OperationNotPermitted(func_call)
            elif e.errno == PermissionDenied.errno:
                raise PermissionDenied(func_call)
            elif e.errno == ReadOnlyFileSystem.errno:
                raise ReadOnlyFileSystem(func_call)
            else:
                raise
        else:
            myfd_st = None
            try:
                myfd_st = os.fstat(myfd)
                if not preexisting:
                    # Don't chown the file if it is preexisting, since we
                    # want to preserve existing permissions in that case.
                    if portage.data.secpass >= 1 and myfd_st.st_gid != portage_gid:
                        os.fchown(myfd, -1, portage_gid)
            except OSError as e:
                if e.errno not in (errno.ENOENT, errno.ESTALE):
                    writemsg("%s: fchown('%s', -1, %d)\n" % \
                     (e, lockfilename, portage_gid), noiselevel=-1)
                    writemsg(_("Cannot chown a lockfile: '%s'\n") % \
                     lockfilename, noiselevel=-1)
                    writemsg(_("Group IDs of current user: %s\n") % \
                     " ".join(str(n) for n in os.getgroups()),
                     noiselevel=-1)
                else:
                    # another process has removed the file, so we'll have
                    # to create it again
                    continue
            finally:
                os.close(myfd)

            # If fstat shows more than one hardlink, then it's extremely
            # unlikely that the following link call will result in a lock,
            # so optimize away the wasteful link call and sleep or raise
            # TryAgain.
            if myfd_st is not None and myfd_st.st_nlink < 2:
                try:
                    os.link(lockfilename, myhardlock)
                except OSError as e:
                    func_call = "link('%s', '%s')" % (lockfilename, myhardlock)
                    if e.errno == OperationNotPermitted.errno:
                        raise OperationNotPermitted(func_call)
                    elif e.errno == PermissionDenied.errno:
                        raise PermissionDenied(func_call)
                    elif e.errno in (errno.ESTALE, errno.ENOENT):
                        # another process has removed the file, so we'll have
                        # to create it again
                        continue
                    else:
                        raise
                else:
                    if hardlink_is_mine(myhardlock, lockfilename):
                        if out is not None:
                            out.eend(os.EX_OK)
                        break

                    try:
                        os.unlink(myhardlock)
                    except OSError as e:
                        # This should not happen, since the file name of
                        # myhardlock is unique to our host and PID,
                        # and the above link() call succeeded.
                        if e.errno not in (errno.ENOENT, errno.ESTALE):
                            raise
                        raise FileNotFound(myhardlock)

        if flags & os.O_NONBLOCK:
            raise TryAgain(lockfilename)

        if out is None and not _quiet:
            out = portage.output.EOutput()
        if out is not None and not displayed_waiting_msg:
            displayed_waiting_msg = True
            if waiting_msg is None:
                waiting_msg = _("waiting for lock on %s\n") % lockfilename
            out.ebegin(waiting_msg)

        time.sleep(_HARDLINK_POLL_LATENCY)

    return True
Exemple #33
0
def unlockfile(mytuple):

    #XXX: Compatability hack.
    if len(mytuple) == 3:
        lockfilename, myfd, unlinkfile = mytuple
        locking_method = fcntl.flock
    elif len(mytuple) == 4:
        lockfilename, myfd, unlinkfile, locking_method = mytuple
    else:
        raise InvalidData

    if (myfd == HARDLINK_FD):
        unhardlink_lockfile(lockfilename, unlinkfile=unlinkfile)
        return True

    # myfd may be None here due to myfd = mypath in lockfile()
    if isinstance(lockfilename, basestring) and \
     not os.path.exists(lockfilename):
        writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
        if myfd is not None:
            os.close(myfd)
            _open_fds.remove(myfd)
        return False

    try:
        if myfd is None:
            myfd = os.open(lockfilename, os.O_WRONLY, 0o660)
            unlinkfile = 1
        locking_method(myfd, fcntl.LOCK_UN)
    except OSError:
        if isinstance(lockfilename, basestring):
            os.close(myfd)
            _open_fds.remove(myfd)
        raise IOError(_("Failed to unlock file '%s'\n") % lockfilename)

    try:
        # This sleep call was added to allow other processes that are
        # waiting for a lock to be able to grab it before it is deleted.
        # lockfile() already accounts for this situation, however, and
        # the sleep here adds more time than is saved overall, so am
        # commenting until it is proved necessary.
        #time.sleep(0.0001)
        if unlinkfile:
            locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
            # We won the lock, so there isn't competition for it.
            # We can safely delete the file.
            writemsg(_("Got the lockfile...\n"), 1)
            if _fstat_nlink(myfd) == 1:
                os.unlink(lockfilename)
                writemsg(_("Unlinked lockfile...\n"), 1)
                locking_method(myfd, fcntl.LOCK_UN)
            else:
                writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
                os.close(myfd)
                _open_fds.remove(myfd)
                return False
    except SystemExit:
        raise
    except Exception as e:
        writemsg(_("Failed to get lock... someone took it.\n"), 1)
        writemsg(str(e) + "\n", 1)

    # why test lockfilename?  because we may have been handed an
    # fd originally, and the caller might not like having their
    # open fd closed automatically on them.
    if isinstance(lockfilename, basestring):
        os.close(myfd)
        _open_fds.remove(myfd)

    return True
Exemple #34
0
def _lockfile_iteration(mypath,
                        wantnewlockfile=False,
                        unlinkfile=False,
                        waiting_msg=None,
                        flags=0):
    """
	Acquire a lock on mypath, without retry. Return None if the lockfile
	was removed by previous lock holder (caller must retry).

	@param mypath: lock file path
	@type mypath: str
	@param wantnewlockfile: use a separate new lock file
	@type wantnewlockfile: bool
	@param unlinkfile: remove lock file prior to unlock
	@type unlinkfile: bool
	@param waiting_msg: message to show before blocking
	@type waiting_msg: str
	@param flags: lock flags (only supports os.O_NONBLOCK)
	@type flags: int
	@rtype: bool
	@return: unlockfile tuple on success, None if retry is needed
	"""
    if not mypath:
        raise InvalidData(_("Empty path given"))

    # Since Python 3.4, chown requires int type (no proxies).
    portage_gid = int(portage.data.portage_gid)

    # Support for file object or integer file descriptor parameters is
    # deprecated due to ambiguity in whether or not it's safe to close
    # the file descriptor, making it prone to "Bad file descriptor" errors
    # or file descriptor leaks.
    if isinstance(mypath, basestring) and mypath[-1] == '/':
        mypath = mypath[:-1]

    lockfilename_path = mypath
    if hasattr(mypath, 'fileno'):
        warnings.warn(
            "portage.locks.lockfile() support for "
            "file object parameters is deprecated. Use a file path instead.",
            DeprecationWarning,
            stacklevel=2)
        lockfilename_path = getattr(mypath, 'name', None)
        mypath = mypath.fileno()
    if isinstance(mypath, int):
        warnings.warn(
            "portage.locks.lockfile() support for integer file "
            "descriptor parameters is deprecated. Use a file path instead.",
            DeprecationWarning,
            stacklevel=2)
        lockfilename = mypath
        wantnewlockfile = 0
        unlinkfile = 0
    elif wantnewlockfile:
        base, tail = os.path.split(mypath)
        lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
        lockfilename_path = lockfilename
        unlinkfile = 1
    else:
        lockfilename = mypath

    if isinstance(mypath, basestring):
        if not os.path.exists(os.path.dirname(mypath)):
            raise DirectoryNotFound(os.path.dirname(mypath))
        preexisting = os.path.exists(lockfilename)
        old_mask = os.umask(000)
        try:
            while True:
                try:
                    myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660)
                except OSError as e:
                    if e.errno in (errno.ENOENT,
                                   errno.ESTALE) and os.path.isdir(
                                       os.path.dirname(lockfilename)):
                        # Retry required for NFS (see bug 636798).
                        continue
                    else:
                        _raise_exc(e)
                else:
                    break

            if not preexisting:
                try:
                    if portage.data.secpass >= 1 and os.stat(
                            lockfilename).st_gid != portage_gid:
                        os.chown(lockfilename, -1, portage_gid)
                except OSError as e:
                    if e.errno in (errno.ENOENT, errno.ESTALE):
                        os.close(myfd)
                        return None
                    else:
                        writemsg("%s: chown('%s', -1, %d)\n" % \
                         (e, lockfilename, portage_gid), noiselevel=-1)
                        writemsg(_("Cannot chown a lockfile: '%s'\n") % \
                         lockfilename, noiselevel=-1)
                        writemsg(_("Group IDs of current user: %s\n") % \
                         " ".join(str(n) for n in os.getgroups()),
                         noiselevel=-1)
        finally:
            os.umask(old_mask)

    elif isinstance(mypath, int):
        myfd = mypath

    else:
        raise ValueError(_("Unknown type passed in '%s': '%s'") % \
         (type(mypath), mypath))

    # try for a non-blocking lock, if it's held, throw a message
    # we're waiting on lockfile and use a blocking attempt.
    locking_method = portage._eintr_func_wrapper(_get_lock_fn())
    try:
        if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
            raise IOError(errno.ENOSYS, "Function not implemented")
        locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
    except IOError as e:
        if not hasattr(e, "errno"):
            raise
        if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK):
            # resource temp unavailable; eg, someone beat us to the lock.
            if flags & os.O_NONBLOCK:
                os.close(myfd)
                raise TryAgain(mypath)

            global _quiet
            if _quiet:
                out = None
            else:
                out = portage.output.EOutput()
            if waiting_msg is None:
                if isinstance(mypath, int):
                    waiting_msg = _("waiting for lock on fd %i") % myfd
                else:
                    waiting_msg = _("waiting for lock on %s") % lockfilename
            if out is not None:
                out.ebegin(waiting_msg)
            # try for the exclusive lock now.
            enolock_msg_shown = False
            while True:
                try:
                    locking_method(myfd, fcntl.LOCK_EX)
                except EnvironmentError as e:
                    if e.errno == errno.ENOLCK:
                        # This is known to occur on Solaris NFS (see
                        # bug #462694). Assume that the error is due
                        # to temporary exhaustion of record locks,
                        # and loop until one becomes available.
                        if not enolock_msg_shown:
                            enolock_msg_shown = True
                            if isinstance(mypath, int):
                                context_desc = _("Error while waiting "
                                                 "to lock fd %i") % myfd
                            else:
                                context_desc = _("Error while waiting "
                                                 "to lock '%s'") % lockfilename
                            writemsg("\n!!! %s: %s\n" % (context_desc, e),
                                     noiselevel=-1)

                        time.sleep(_HARDLINK_POLL_LATENCY)
                        continue

                    if out is not None:
                        out.eend(1, str(e))
                    raise
                else:
                    break

            if out is not None:
                out.eend(os.EX_OK)
        elif e.errno in (errno.ENOSYS, ):
            # We're not allowed to lock on this FS.
            if not isinstance(lockfilename, int):
                # If a file object was passed in, it's not safe
                # to close the file descriptor because it may
                # still be in use.
                os.close(myfd)
            lockfilename_path = _unicode_decode(lockfilename_path,
                                                encoding=_encodings['fs'],
                                                errors='strict')
            if not isinstance(lockfilename_path, basestring):
                raise
            link_success = hardlink_lockfile(lockfilename_path,
                                             waiting_msg=waiting_msg,
                                             flags=flags)
            if not link_success:
                raise
            lockfilename = lockfilename_path
            locking_method = None
            myfd = HARDLINK_FD
        else:
            raise

    if isinstance(lockfilename,
                  basestring) and myfd != HARDLINK_FD and unlinkfile:
        try:
            removed = _lockfile_was_removed(myfd, lockfilename)
        except Exception:
            # Do not leak the file descriptor here.
            os.close(myfd)
            raise
        else:
            if removed:
                # Removed by previous lock holder... Caller will retry...
                os.close(myfd)
                return None

    if myfd != HARDLINK_FD:

        # FD_CLOEXEC is enabled by default in Python >=3.4.
        if sys.hexversion < 0x3040000:
            try:
                fcntl.FD_CLOEXEC
            except AttributeError:
                pass
            else:
                fcntl.fcntl(
                    myfd, fcntl.F_SETFD,
                    fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)

        _open_fds.add(myfd)

    writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1)
    return (lockfilename, myfd, unlinkfile, locking_method)
    def _start(self):

        need_builddir = self.phase not in self._phases_without_builddir

        # This can happen if the pre-clean phase triggers
        # die_hooks for some reason, and PORTAGE_BUILDDIR
        # doesn't exist yet.
        if need_builddir and \
         not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
            msg = _("The ebuild phase '%s' has been aborted "
            "since PORTAGE_BUILDDIR does not exist: '%s'") % \
            (self.phase, self.settings['PORTAGE_BUILDDIR'])
            self._eerror(textwrap.wrap(msg, 72))
            self._set_returncode((self.pid, 1 << 8))
            self._async_wait()
            return

        # Check if the cgroup hierarchy is in place. If it's not, mount it.
        if (os.geteuid() == 0 and platform.system() == 'Linux'
                and 'cgroup' in self.settings.features
                and self.phase not in self._phases_without_cgroup):
            cgroup_root = '/sys/fs/cgroup'
            cgroup_portage = os.path.join(cgroup_root, 'portage')

            try:
                # cgroup tmpfs
                if not os.path.ismount(cgroup_root):
                    # we expect /sys/fs to be there already
                    if not os.path.isdir(cgroup_root):
                        os.mkdir(cgroup_root, 0o755)
                    subprocess.check_call([
                        'mount', '-t', 'tmpfs', '-o',
                        'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs',
                        cgroup_root
                    ])

                # portage subsystem
                if not os.path.ismount(cgroup_portage):
                    if not os.path.isdir(cgroup_portage):
                        os.mkdir(cgroup_portage, 0o755)
                    subprocess.check_call([
                        'mount', '-t', 'cgroup', '-o',
                        'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs',
                        cgroup_portage
                    ])

                cgroup_path = tempfile.mkdtemp(
                    dir=cgroup_portage,
                    prefix='%s:%s.' %
                    (self.settings["CATEGORY"], self.settings["PF"]))
            except (subprocess.CalledProcessError, OSError):
                pass
            else:
                self.cgroup = cgroup_path

        if self.background:
            # Automatically prevent color codes from showing up in logs,
            # since we're not displaying to a terminal anyway.
            self.settings['NOCOLOR'] = 'true'

        if self._enable_ipc_daemon:
            self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
            if self.phase not in self._phases_without_builddir:
                if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings:
                    self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
                                                     settings=self.settings)
                    self._build_dir.lock()
                self.settings['PORTAGE_IPC_DAEMON'] = "1"
                self._start_ipc_daemon()
            else:
                self.settings.pop('PORTAGE_IPC_DAEMON', None)
        else:
            # Since the IPC daemon is disabled, use a simple tempfile based
            # approach to detect unexpected exit like in bug #190128.
            self.settings.pop('PORTAGE_IPC_DAEMON', None)
            if self.phase not in self._phases_without_builddir:
                exit_file = os.path.join(self.settings['PORTAGE_BUILDDIR'],
                                         '.exit_status')
                self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
                try:
                    os.unlink(exit_file)
                except OSError:
                    if os.path.exists(exit_file):
                        # make sure it doesn't exist
                        raise
            else:
                self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)

        if self.fd_pipes is None:
            self.fd_pipes = {}
        null_fd = None
        if 0 not in self.fd_pipes and \
         self.phase not in self._phases_interactive_whitelist and \
         "interactive" not in self.settings.get("PROPERTIES", "").split():
            null_fd = os.open('/dev/null', os.O_RDONLY)
            self.fd_pipes[0] = null_fd

        try:
            SpawnProcess._start(self)
        finally:
            if null_fd is not None:
                os.close(null_fd)
Exemple #36
0
def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
	waiting_msg=None, flags=0):
	"""
	If wantnewlockfile is True then this creates a lockfile in the parent
	directory as the file: '.' + basename + '.portage_lockfile'.
	"""

	if not mypath:
		raise InvalidData(_("Empty path given"))

	# Since Python 3.4, chown requires int type (no proxies).
	portage_gid = int(portage.data.portage_gid)

	# Support for file object or integer file descriptor parameters is
	# deprecated due to ambiguity in whether or not it's safe to close
	# the file descriptor, making it prone to "Bad file descriptor" errors
	# or file descriptor leaks.
	if isinstance(mypath, basestring) and mypath[-1] == '/':
		mypath = mypath[:-1]

	lockfilename_path = mypath
	if hasattr(mypath, 'fileno'):
		warnings.warn("portage.locks.lockfile() support for "
			"file object parameters is deprecated. Use a file path instead.",
			DeprecationWarning, stacklevel=2)
		lockfilename_path = getattr(mypath, 'name', None)
		mypath = mypath.fileno()
	if isinstance(mypath, int):
		warnings.warn("portage.locks.lockfile() support for integer file "
			"descriptor parameters is deprecated. Use a file path instead.",
			DeprecationWarning, stacklevel=2)
		lockfilename    = mypath
		wantnewlockfile = 0
		unlinkfile      = 0
	elif wantnewlockfile:
		base, tail = os.path.split(mypath)
		lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
		lockfilename_path = lockfilename
		unlinkfile   = 1
	else:
		lockfilename = mypath

	if isinstance(mypath, basestring):
		if not os.path.exists(os.path.dirname(mypath)):
			raise DirectoryNotFound(os.path.dirname(mypath))
		preexisting = os.path.exists(lockfilename)
		old_mask = os.umask(000)
		try:
			try:
				myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
			except OSError as e:
				func_call = "open('%s')" % lockfilename
				if e.errno == OperationNotPermitted.errno:
					raise OperationNotPermitted(func_call)
				elif e.errno == PermissionDenied.errno:
					raise PermissionDenied(func_call)
				elif e.errno == ReadOnlyFileSystem.errno:
					raise ReadOnlyFileSystem(func_call)
				else:
					raise

			if not preexisting:
				try:
					if os.stat(lockfilename).st_gid != portage_gid:
						os.chown(lockfilename, -1, portage_gid)
				except OSError as e:
					if e.errno in (errno.ENOENT, errno.ESTALE):
						return lockfile(mypath,
							wantnewlockfile=wantnewlockfile,
							unlinkfile=unlinkfile, waiting_msg=waiting_msg,
							flags=flags)
					else:
						writemsg("%s: chown('%s', -1, %d)\n" % \
							(e, lockfilename, portage_gid), noiselevel=-1)
						writemsg(_("Cannot chown a lockfile: '%s'\n") % \
							lockfilename, noiselevel=-1)
						writemsg(_("Group IDs of current user: %s\n") % \
							" ".join(str(n) for n in os.getgroups()),
							noiselevel=-1)
		finally:
			os.umask(old_mask)

	elif isinstance(mypath, int):
		myfd = mypath

	else:
		raise ValueError(_("Unknown type passed in '%s': '%s'") % \
			(type(mypath), mypath))

	# try for a non-blocking lock, if it's held, throw a message
	# we're waiting on lockfile and use a blocking attempt.
	locking_method = portage._eintr_func_wrapper(_default_lock_fn)
	try:
		if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
			raise IOError(errno.ENOSYS, "Function not implemented")
		locking_method(myfd, fcntl.LOCK_EX|fcntl.LOCK_NB)
	except IOError as e:
		if not hasattr(e, "errno"):
			raise
		if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK):
			# resource temp unavailable; eg, someone beat us to the lock.
			if flags & os.O_NONBLOCK:
				os.close(myfd)
				raise TryAgain(mypath)

			global _quiet
			if _quiet:
				out = None
			else:
				out = portage.output.EOutput()
			if waiting_msg is None:
				if isinstance(mypath, int):
					waiting_msg = _("waiting for lock on fd %i") % myfd
				else:
					waiting_msg = _("waiting for lock on %s") % lockfilename
			if out is not None:
				out.ebegin(waiting_msg)
			# try for the exclusive lock now.
			enolock_msg_shown = False
			while True:
				try:
					locking_method(myfd, fcntl.LOCK_EX)
				except EnvironmentError as e:
					if e.errno == errno.ENOLCK:
						# This is known to occur on Solaris NFS (see
						# bug #462694). Assume that the error is due
						# to temporary exhaustion of record locks,
						# and loop until one becomes available.
						if not enolock_msg_shown:
							enolock_msg_shown = True
							if isinstance(mypath, int):
								context_desc = _("Error while waiting "
									"to lock fd %i") % myfd
							else:
								context_desc = _("Error while waiting "
									"to lock '%s'") % lockfilename
							writemsg("\n!!! %s: %s\n" % (context_desc, e),
								noiselevel=-1)

						time.sleep(_HARDLINK_POLL_LATENCY)
						continue

					if out is not None:
						out.eend(1, str(e))
					raise
				else:
					break

			if out is not None:
				out.eend(os.EX_OK)
		elif e.errno in (errno.ENOSYS,):
			# We're not allowed to lock on this FS.
			if not isinstance(lockfilename, int):
				# If a file object was passed in, it's not safe
				# to close the file descriptor because it may
				# still be in use.
				os.close(myfd)
			lockfilename_path = _unicode_decode(lockfilename_path,
				encoding=_encodings['fs'], errors='strict')
			if not isinstance(lockfilename_path, basestring):
				raise
			link_success = hardlink_lockfile(lockfilename_path,
				waiting_msg=waiting_msg, flags=flags)
			if not link_success:
				raise
			lockfilename = lockfilename_path
			locking_method = None
			myfd = HARDLINK_FD
		else:
			raise

		
	if isinstance(lockfilename, basestring) and \
		myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
		# The file was deleted on us... Keep trying to make one...
		os.close(myfd)
		writemsg(_("lockfile recurse\n"), 1)
		lockfilename, myfd, unlinkfile, locking_method = lockfile(
			mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
			waiting_msg=waiting_msg, flags=flags)

	if myfd != HARDLINK_FD:

		# FD_CLOEXEC is enabled by default in Python >=3.4.
		if sys.hexversion < 0x3040000:
			try:
				fcntl.FD_CLOEXEC
			except AttributeError:
				pass
			else:
				fcntl.fcntl(myfd, fcntl.F_SETFD,
					fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)

		_open_fds.add(myfd)

	writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1)
	return (lockfilename, myfd, unlinkfile, locking_method)
Exemple #37
0
def unlockfile(mytuple):

	#XXX: Compatability hack.
	if len(mytuple) == 3:
		lockfilename, myfd, unlinkfile = mytuple
		locking_method = fcntl.flock
	elif len(mytuple) == 4:
		lockfilename, myfd, unlinkfile, locking_method = mytuple
	else:
		raise InvalidData

	if(myfd == HARDLINK_FD):
		unhardlink_lockfile(lockfilename, unlinkfile=unlinkfile)
		return True
	
	# myfd may be None here due to myfd = mypath in lockfile()
	if isinstance(lockfilename, basestring) and \
		not os.path.exists(lockfilename):
		writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
		if myfd is not None:
			os.close(myfd)
			_open_fds.remove(myfd)
		return False

	try:
		if myfd is None:
			myfd = os.open(lockfilename, os.O_WRONLY, 0o660)
			unlinkfile = 1
		locking_method(myfd, fcntl.LOCK_UN)
	except OSError:
		if isinstance(lockfilename, basestring):
			os.close(myfd)
			_open_fds.remove(myfd)
		raise IOError(_("Failed to unlock file '%s'\n") % lockfilename)

	try:
		# This sleep call was added to allow other processes that are
		# waiting for a lock to be able to grab it before it is deleted.
		# lockfile() already accounts for this situation, however, and
		# the sleep here adds more time than is saved overall, so am
		# commenting until it is proved necessary.
		#time.sleep(0.0001)
		if unlinkfile:
			locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
			# We won the lock, so there isn't competition for it.
			# We can safely delete the file.
			writemsg(_("Got the lockfile...\n"), 1)
			if _fstat_nlink(myfd) == 1:
				os.unlink(lockfilename)
				writemsg(_("Unlinked lockfile...\n"), 1)
				locking_method(myfd, fcntl.LOCK_UN)
			else:
				writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
				os.close(myfd)
				_open_fds.remove(myfd)
				return False
	except SystemExit:
		raise
	except Exception as e:
		writemsg(_("Failed to get lock... someone took it.\n"), 1)
		writemsg(str(e) + "\n", 1)

	# why test lockfilename?  because we may have been handed an
	# fd originally, and the caller might not like having their
	# open fd closed automatically on them.
	if isinstance(lockfilename, basestring):
		os.close(myfd)
		_open_fds.remove(myfd)

	return True
Exemple #38
0
	def _start(self):

		if self.fd_pipes is None:
			self.fd_pipes = {}
		fd_pipes = self.fd_pipes

		self._files = self._files_dict()
		files = self._files

		master_fd, slave_fd = self._pipe(fd_pipes)
		fcntl.fcntl(master_fd, fcntl.F_SETFL,
			fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
		files.process = master_fd

		logfile = None
		if self._can_log(slave_fd):
			logfile = self.logfile

		null_input = None
		if not self.background or 0 in fd_pipes:
			# Subclasses such as AbstractEbuildProcess may have already passed
			# in a null file descriptor in fd_pipes, so use that when given.
			pass
		else:
			# TODO: Use job control functions like tcsetpgrp() to control
			# access to stdin. Until then, use /dev/null so that any
			# attempts to read from stdin will immediately return EOF
			# instead of blocking indefinitely.
			null_input = os.open('/dev/null', os.O_RDWR)
			fd_pipes[0] = null_input

		fd_pipes.setdefault(0, sys.stdin.fileno())
		fd_pipes.setdefault(1, sys.stdout.fileno())
		fd_pipes.setdefault(2, sys.stderr.fileno())

		# flush any pending output
		for fd in fd_pipes.values():
			if fd == sys.stdout.fileno():
				sys.stdout.flush()
			if fd == sys.stderr.fileno():
				sys.stderr.flush()

		if logfile is not None:

			fd_pipes_orig = fd_pipes.copy()
			fd_pipes[1] = slave_fd
			fd_pipes[2] = slave_fd

			files.log = open(_unicode_encode(logfile,
				encoding=_encodings['fs'], errors='strict'), mode='ab')
			if logfile.endswith('.gz'):
				self._log_file_real = files.log
				files.log = gzip.GzipFile(filename='', mode='ab',
					fileobj=files.log)

			portage.util.apply_secpass_permissions(logfile,
				uid=portage.portage_uid, gid=portage.portage_gid,
				mode=0o660)

			if not self.background:
				files.stdout = os.dup(fd_pipes_orig[1])

			output_handler = self._output_handler

		else:

			# Create a dummy pipe so the scheduler can monitor
			# the process from inside a poll() loop.
			fd_pipes[self._dummy_pipe_fd] = slave_fd
			if self.background:
				fd_pipes[1] = slave_fd
				fd_pipes[2] = slave_fd
			output_handler = self._dummy_handler

		kwargs = {}
		for k in self._spawn_kwarg_names:
			v = getattr(self, k)
			if v is not None:
				kwargs[k] = v

		kwargs["fd_pipes"] = fd_pipes
		kwargs["returnpid"] = True
		kwargs.pop("logfile", None)

		self._reg_id = self.scheduler.register(files.process,
			self._registered_events, output_handler)
		self._registered = True

		retval = self._spawn(self.args, **kwargs)

		os.close(slave_fd)
		if null_input is not None:
			os.close(null_input)

		if isinstance(retval, int):
			# spawn failed
			self._unregister()
			self._set_returncode((self.pid, retval))
			self.wait()
			return

		self.pid = retval[0]
		portage.process.spawned_pids.remove(self.pid)
Exemple #39
0
	def _start(self):

		if self.fd_pipes is None:
			self.fd_pipes = {}
		fd_pipes = self.fd_pipes

		master_fd, slave_fd = self._pipe(fd_pipes)

		can_log = self._can_log(slave_fd)
		if can_log:
			log_file_path = self.logfile
		else:
			log_file_path = None

		null_input = None
		if not self.background or 0 in fd_pipes:
			# Subclasses such as AbstractEbuildProcess may have already passed
			# in a null file descriptor in fd_pipes, so use that when given.
			pass
		else:
			# TODO: Use job control functions like tcsetpgrp() to control
			# access to stdin. Until then, use /dev/null so that any
			# attempts to read from stdin will immediately return EOF
			# instead of blocking indefinitely.
			null_input = os.open('/dev/null', os.O_RDWR)
			fd_pipes[0] = null_input

		fd_pipes.setdefault(0, sys.__stdin__.fileno())
		fd_pipes.setdefault(1, sys.__stdout__.fileno())
		fd_pipes.setdefault(2, sys.__stderr__.fileno())

		# flush any pending output
		stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
		for fd in fd_pipes.values():
			if fd in stdout_filenos:
				sys.__stdout__.flush()
				sys.__stderr__.flush()
				break

		fd_pipes_orig = fd_pipes.copy()

		if log_file_path is not None:
			fd_pipes[1] = slave_fd
			fd_pipes[2] = slave_fd

		else:
			# Create a dummy pipe so the scheduler can monitor
			# the process from inside a poll() loop.
			fd_pipes[self._dummy_pipe_fd] = slave_fd
			if self.background:
				fd_pipes[1] = slave_fd
				fd_pipes[2] = slave_fd

		kwargs = {}
		for k in self._spawn_kwarg_names:
			v = getattr(self, k)
			if v is not None:
				kwargs[k] = v

		kwargs["fd_pipes"] = fd_pipes
		kwargs["returnpid"] = True
		kwargs.pop("logfile", None)

		retval = self._spawn(self.args, **kwargs)

		os.close(slave_fd)
		if null_input is not None:
			os.close(null_input)

		if isinstance(retval, int):
			# spawn failed
			self._unregister()
			self._set_returncode((self.pid, retval))
			self.wait()
			return

		self.pid = retval[0]
		portage.process.spawned_pids.remove(self.pid)

		stdout_fd = None
		if can_log and not self.background:
			stdout_fd = os.dup(fd_pipes_orig[1])

		self._pipe_logger = PipeLogger(background=self.background,
			scheduler=self.scheduler, input_fd=master_fd,
			log_file_path=log_file_path,
			stdout_fd=stdout_fd)
		self._pipe_logger.addExitListener(self._pipe_logger_exit)
		self._pipe_logger.start()
		self._registered = True
Exemple #40
0
def lockfile(mypath,
             wantnewlockfile=0,
             unlinkfile=0,
             waiting_msg=None,
             flags=0):
    """
	If wantnewlockfile is True then this creates a lockfile in the parent
	directory as the file: '.' + basename + '.portage_lockfile'.
	"""

    if not mypath:
        raise InvalidData(_("Empty path given"))

    # Support for file object or integer file descriptor parameters is
    # deprecated due to ambiguity in whether or not it's safe to close
    # the file descriptor, making it prone to "Bad file descriptor" errors
    # or file descriptor leaks.
    if isinstance(mypath, basestring) and mypath[-1] == '/':
        mypath = mypath[:-1]

    lockfilename_path = mypath
    if hasattr(mypath, 'fileno'):
        warnings.warn(
            "portage.locks.lockfile() support for "
            "file object parameters is deprecated. Use a file path instead.",
            DeprecationWarning,
            stacklevel=2)
        lockfilename_path = getattr(mypath, 'name', None)
        mypath = mypath.fileno()
    if isinstance(mypath, int):
        warnings.warn(
            "portage.locks.lockfile() support for integer file "
            "descriptor parameters is deprecated. Use a file path instead.",
            DeprecationWarning,
            stacklevel=2)
        lockfilename = mypath
        wantnewlockfile = 0
        unlinkfile = 0
    elif wantnewlockfile:
        base, tail = os.path.split(mypath)
        lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
        lockfilename_path = lockfilename
        unlinkfile = 1
    else:
        lockfilename = mypath

    if isinstance(mypath, basestring):
        if not os.path.exists(os.path.dirname(mypath)):
            raise DirectoryNotFound(os.path.dirname(mypath))
        preexisting = os.path.exists(lockfilename)
        old_mask = os.umask(000)
        try:
            try:
                myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660)
            except OSError as e:
                func_call = "open('%s')" % lockfilename
                if e.errno == OperationNotPermitted.errno:
                    raise OperationNotPermitted(func_call)
                elif e.errno == PermissionDenied.errno:
                    raise PermissionDenied(func_call)
                else:
                    raise

            if not preexisting:
                try:
                    if os.stat(lockfilename).st_gid != portage_gid:
                        os.chown(lockfilename, -1, portage_gid)
                except OSError as e:
                    if e.errno in (errno.ENOENT, errno.ESTALE):
                        return lockfile(mypath,
                                        wantnewlockfile=wantnewlockfile,
                                        unlinkfile=unlinkfile,
                                        waiting_msg=waiting_msg,
                                        flags=flags)
                    else:
                        writemsg("%s: chown('%s', -1, %d)\n" % \
                         (e, lockfilename, portage_gid), noiselevel=-1)
                        writemsg(_("Cannot chown a lockfile: '%s'\n") % \
                         lockfilename, noiselevel=-1)
                        writemsg(_("Group IDs of current user: %s\n") % \
                         " ".join(str(n) for n in os.getgroups()),
                         noiselevel=-1)
        finally:
            os.umask(old_mask)

    elif isinstance(mypath, int):
        myfd = mypath

    else:
        raise ValueError(_("Unknown type passed in '%s': '%s'") % \
         (type(mypath), mypath))

    # try for a non-blocking lock, if it's held, throw a message
    # we're waiting on lockfile and use a blocking attempt.
    locking_method = _default_lock_fn
    try:
        if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
            raise IOError(errno.ENOSYS, "Function not implemented")
        locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
    except IOError as e:
        if not hasattr(e, "errno"):
            raise
        if e.errno in (errno.EACCES, errno.EAGAIN):
            # resource temp unavailable; eg, someone beat us to the lock.
            if flags & os.O_NONBLOCK:
                os.close(myfd)
                raise TryAgain(mypath)

            global _quiet
            if _quiet:
                out = None
            else:
                out = portage.output.EOutput()
            if waiting_msg is None:
                if isinstance(mypath, int):
                    waiting_msg = _("waiting for lock on fd %i") % myfd
                else:
                    waiting_msg = _("waiting for lock on %s\n") % lockfilename
            if out is not None:
                out.ebegin(waiting_msg)
            # try for the exclusive lock now.
            try:
                locking_method(myfd, fcntl.LOCK_EX)
            except EnvironmentError as e:
                if out is not None:
                    out.eend(1, str(e))
                raise
            if out is not None:
                out.eend(os.EX_OK)
        elif e.errno in (errno.ENOSYS, errno.ENOLCK):
            # We're not allowed to lock on this FS.
            if not isinstance(lockfilename, int):
                # If a file object was passed in, it's not safe
                # to close the file descriptor because it may
                # still be in use.
                os.close(myfd)
            lockfilename_path = _unicode_decode(lockfilename_path,
                                                encoding=_encodings['fs'],
                                                errors='strict')
            if not isinstance(lockfilename_path, basestring):
                raise
            link_success = hardlink_lockfile(lockfilename_path,
                                             waiting_msg=waiting_msg,
                                             flags=flags)
            if not link_success:
                raise
            lockfilename = lockfilename_path
            locking_method = None
            myfd = HARDLINK_FD
        else:
            raise


    if isinstance(lockfilename, basestring) and \
     myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
        # The file was deleted on us... Keep trying to make one...
        os.close(myfd)
        writemsg(_("lockfile recurse\n"), 1)
        lockfilename, myfd, unlinkfile, locking_method = lockfile(
            mypath,
            wantnewlockfile=wantnewlockfile,
            unlinkfile=unlinkfile,
            waiting_msg=waiting_msg,
            flags=flags)

    if myfd != HARDLINK_FD:
        _open_fds.add(myfd)

    writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1)
    return (lockfilename, myfd, unlinkfile, locking_method)
	def _send_reply(self, reply):
		output_fd = os.open(self.output_fifo, os.O_WRONLY|os.O_NONBLOCK)
		output_file = os.fdopen(output_fd, 'wb')
		pickle.dump(reply, output_file)
		output_file.close()
Exemple #42
0
	def _start(self):

		need_builddir = self.phase not in self._phases_without_builddir

		# This can happen if the pre-clean phase triggers
		# die_hooks for some reason, and PORTAGE_BUILDDIR
		# doesn't exist yet.
		if need_builddir and \
			not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
			msg = _("The ebuild phase '%s' has been aborted "
			"since PORTAGE_BUILDIR does not exist: '%s'") % \
			(self.phase, self.settings['PORTAGE_BUILDDIR'])
			self._eerror(textwrap.wrap(msg, 72))
			self._set_returncode((self.pid, 1 << 8))
			self.wait()
			return

		if self.background:
			# Automatically prevent color codes from showing up in logs,
			# since we're not displaying to a terminal anyway.
			self.settings['NOCOLOR'] = 'true'

		if self._enable_ipc_daemon:
			self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
			if self.phase not in self._phases_without_builddir:
				if 'PORTAGE_BUILDIR_LOCKED' not in self.settings:
					self._build_dir = EbuildBuildDir(
						scheduler=self.scheduler, settings=self.settings)
					self._build_dir.lock()
				self.settings['PORTAGE_IPC_DAEMON'] = "1"
				self._start_ipc_daemon()
			else:
				self.settings.pop('PORTAGE_IPC_DAEMON', None)
		else:
			# Since the IPC daemon is disabled, use a simple tempfile based
			# approach to detect unexpected exit like in bug #190128.
			self.settings.pop('PORTAGE_IPC_DAEMON', None)
			if self.phase not in self._phases_without_builddir:
				exit_file = os.path.join(
					self.settings['PORTAGE_BUILDDIR'],
					'.exit_status')
				self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
				try:
					os.unlink(exit_file)
				except OSError:
					if os.path.exists(exit_file):
						# make sure it doesn't exist
						raise
			else:
				self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)

		if self.fd_pipes is None:
			self.fd_pipes = {}
		null_fd = None
		if 0 not in self.fd_pipes and \
			self.phase not in self._phases_interactive_whitelist and \
			"interactive" not in self.settings.get("PROPERTIES", "").split():
			null_fd = os.open('/dev/null', os.O_RDONLY)
			self.fd_pipes[0] = null_fd

		try:
			SpawnProcess._start(self)
		finally:
			if null_fd is not None:
				os.close(null_fd)
Exemple #43
0
 def _send_reply(self, reply):
     output_fd = os.open(self.output_fifo, os.O_WRONLY | os.O_NONBLOCK)
     output_file = os.fdopen(output_fd, 'wb')
     pickle.dump(reply, output_file)
     output_file.close()
Exemple #44
0
def lockfile(mypath,
             wantnewlockfile=0,
             unlinkfile=0,
             waiting_msg=None,
             flags=0):
    """
	If wantnewlockfile is True then this creates a lockfile in the parent
	directory as the file: '.' + basename + '.portage_lockfile'.
	"""
    import fcntl

    if not mypath:
        raise InvalidData(_("Empty path given"))

    if isinstance(mypath, basestring) and mypath[-1] == '/':
        mypath = mypath[:-1]

    if hasattr(mypath, 'fileno'):
        mypath = mypath.fileno()
    if isinstance(mypath, int):
        lockfilename = mypath
        wantnewlockfile = 0
        unlinkfile = 0
    elif wantnewlockfile:
        base, tail = os.path.split(mypath)
        lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
        del base, tail
        unlinkfile = 1
    else:
        lockfilename = mypath

    if isinstance(mypath, basestring):
        if not os.path.exists(os.path.dirname(mypath)):
            raise DirectoryNotFound(os.path.dirname(mypath))
        preexisting = os.path.exists(lockfilename)
        old_mask = os.umask(000)
        try:
            try:
                myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660)
            except OSError as e:
                func_call = "open('%s')" % lockfilename
                if e.errno == OperationNotPermitted.errno:
                    raise OperationNotPermitted(func_call)
                elif e.errno == PermissionDenied.errno:
                    raise PermissionDenied(func_call)
                else:
                    raise

            if not preexisting:
                try:
                    if os.stat(lockfilename).st_gid != portage_gid:
                        os.chown(lockfilename, -1, portage_gid)
                except OSError as e:
                    if e.errno in (errno.ENOENT, errno.ESTALE):
                        return lockfile(mypath,
                                        wantnewlockfile=wantnewlockfile,
                                        unlinkfile=unlinkfile,
                                        waiting_msg=waiting_msg,
                                        flags=flags)
                    else:
                        writemsg("%s: chown('%s', -1, %d)\n" % \
                         (e, lockfilename, portage_gid), noiselevel=-1)
                        writemsg(_("Cannot chown a lockfile: '%s'\n") % \
                         lockfilename, noiselevel=-1)
                        writemsg(_("Group IDs of current user: %s\n") % \
                         " ".join(str(n) for n in os.getgroups()),
                         noiselevel=-1)
        finally:
            os.umask(old_mask)

    elif isinstance(mypath, int):
        myfd = mypath

    else:
        raise ValueError(_("Unknown type passed in '%s': '%s'") % \
         (type(mypath), mypath))

    # try for a non-blocking lock, if it's held, throw a message
    # we're waiting on lockfile and use a blocking attempt.
    locking_method = fcntl.lockf
    try:
        fcntl.lockf(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
    except IOError as e:
        if "errno" not in dir(e):
            raise
        if e.errno in (errno.EACCES, errno.EAGAIN):
            # resource temp unavailable; eg, someone beat us to the lock.
            if flags & os.O_NONBLOCK:
                raise TryAgain(mypath)

            global _quiet
            out = EOutput()
            out.quiet = _quiet
            if waiting_msg is None:
                if isinstance(mypath, int):
                    waiting_msg = _("waiting for lock on fd %i") % myfd
                else:
                    waiting_msg = _("waiting for lock on %s\n") % lockfilename
            out.ebegin(waiting_msg)
            # try for the exclusive lock now.
            try:
                fcntl.lockf(myfd, fcntl.LOCK_EX)
            except EnvironmentError as e:
                out.eend(1, str(e))
                raise
            out.eend(os.EX_OK)
        elif e.errno == errno.ENOLCK:
            # We're not allowed to lock on this FS.
            os.close(myfd)
            link_success = False
            if lockfilename == str(lockfilename):
                if wantnewlockfile:
                    try:
                        if os.stat(lockfilename)[stat.ST_NLINK] == 1:
                            os.unlink(lockfilename)
                    except OSError:
                        pass
                    link_success = hardlink_lockfile(lockfilename)
            if not link_success:
                raise
            locking_method = None
            myfd = HARDLINK_FD
        else:
            raise


    if isinstance(lockfilename, basestring) and \
     myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
        # The file was deleted on us... Keep trying to make one...
        os.close(myfd)
        writemsg(_("lockfile recurse\n"), 1)
        lockfilename, myfd, unlinkfile, locking_method = lockfile(
            mypath,
            wantnewlockfile=wantnewlockfile,
            unlinkfile=unlinkfile,
            waiting_msg=waiting_msg,
            flags=flags)

    writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1)
    return (lockfilename, myfd, unlinkfile, locking_method)
Exemple #45
0
    def _start(self):

        if self.fd_pipes is None:
            self.fd_pipes = {}
        else:
            self.fd_pipes = self.fd_pipes.copy()
        fd_pipes = self.fd_pipes

        master_fd, slave_fd = self._pipe(fd_pipes)

        can_log = self._can_log(slave_fd)
        if can_log:
            log_file_path = self.logfile
        else:
            log_file_path = None

        null_input = None
        if not self.background or 0 in fd_pipes:
            # Subclasses such as AbstractEbuildProcess may have already passed
            # in a null file descriptor in fd_pipes, so use that when given.
            pass
        else:
            # TODO: Use job control functions like tcsetpgrp() to control
            # access to stdin. Until then, use /dev/null so that any
            # attempts to read from stdin will immediately return EOF
            # instead of blocking indefinitely.
            null_input = os.open('/dev/null', os.O_RDWR)
            fd_pipes[0] = null_input

        fd_pipes.setdefault(0, portage._get_stdin().fileno())
        fd_pipes.setdefault(1, sys.__stdout__.fileno())
        fd_pipes.setdefault(2, sys.__stderr__.fileno())

        # flush any pending output
        stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
        for fd in fd_pipes.values():
            if fd in stdout_filenos:
                sys.__stdout__.flush()
                sys.__stderr__.flush()
                break

        fd_pipes_orig = fd_pipes.copy()

        if log_file_path is not None or self.background:
            fd_pipes[1] = slave_fd
            fd_pipes[2] = slave_fd

        else:
            # Create a dummy pipe that PipeLogger uses to efficiently
            # monitor for process exit by listening for the EOF event.
            # Re-use of the allocated fd number for the key in fd_pipes
            # guarantees that the keys will not collide for similarly
            # allocated pipes which are used by callers such as
            # FileDigester and MergeProcess. See the _setup_pipes
            # docstring for more benefits of this allocation approach.
            self._dummy_pipe_fd = slave_fd
            fd_pipes[slave_fd] = slave_fd

        kwargs = {}
        for k in self._spawn_kwarg_names:
            v = getattr(self, k)
            if v is not None:
                kwargs[k] = v

        kwargs["fd_pipes"] = fd_pipes
        kwargs["returnpid"] = True
        kwargs.pop("logfile", None)

        retval = self._spawn(self.args, **kwargs)

        os.close(slave_fd)
        if null_input is not None:
            os.close(null_input)

        if isinstance(retval, int):
            # spawn failed
            self.returncode = retval
            self._async_wait()
            return

        self.pid = retval[0]

        stdout_fd = None
        if can_log and not self.background:
            stdout_fd = os.dup(fd_pipes_orig[1])

        build_logger = BuildLogger(env=self.env,
                                   log_path=log_file_path,
                                   log_filter_file=self.log_filter_file,
                                   scheduler=self.scheduler)
        build_logger.start()

        pipe_logger = PipeLogger(background=self.background,
                                 scheduler=self.scheduler,
                                 input_fd=master_fd,
                                 log_file_path=build_logger.stdin,
                                 stdout_fd=stdout_fd)

        pipe_logger.start()

        self._registered = True
        self._main_task = asyncio.ensure_future(self._main(
            build_logger, pipe_logger),
                                                loop=self.scheduler)
        self._main_task.add_done_callback(self._main_exit)
def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
	waiting_msg=None, flags=0):
	"""
	If wantnewlockfile is True then this creates a lockfile in the parent
	directory as the file: '.' + basename + '.portage_lockfile'.
	"""
	import fcntl

	if not mypath:
		raise InvalidData(_("Empty path given"))

	if isinstance(mypath, basestring) and mypath[-1] == '/':
		mypath = mypath[:-1]

	if hasattr(mypath, 'fileno'):
		mypath = mypath.fileno()
	if isinstance(mypath, int):
		lockfilename    = mypath
		wantnewlockfile = 0
		unlinkfile      = 0
	elif wantnewlockfile:
		base, tail = os.path.split(mypath)
		lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
		del base, tail
		unlinkfile   = 1
	else:
		lockfilename = mypath

	if isinstance(mypath, basestring):
		if not os.path.exists(os.path.dirname(mypath)):
			raise DirectoryNotFound(os.path.dirname(mypath))
		preexisting = os.path.exists(lockfilename)
		old_mask = os.umask(000)
		try:
			try:
				myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
			except OSError as e:
				func_call = "open('%s')" % lockfilename
				if e.errno == OperationNotPermitted.errno:
					raise OperationNotPermitted(func_call)
				elif e.errno == PermissionDenied.errno:
					raise PermissionDenied(func_call)
				else:
					raise

			if not preexisting:
				try:
					if os.stat(lockfilename).st_gid != portage_gid:
						os.chown(lockfilename, -1, portage_gid)
				except OSError as e:
					if e.errno in (errno.ENOENT, errno.ESTALE):
						return lockfile(mypath,
							wantnewlockfile=wantnewlockfile,
							unlinkfile=unlinkfile, waiting_msg=waiting_msg,
							flags=flags)
					else:
						writemsg(_("Cannot chown a lockfile: '%s'\n") % \
							lockfilename, noiselevel=-1)

		finally:
			os.umask(old_mask)

	elif isinstance(mypath, int):
		myfd = mypath

	else:
		raise ValueError(_("Unknown type passed in '%s': '%s'") % \
			(type(mypath), mypath))

	# try for a non-blocking lock, if it's held, throw a message
	# we're waiting on lockfile and use a blocking attempt.
	locking_method = fcntl.lockf
	try:
		fcntl.lockf(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
	except IOError as e:
		if "errno" not in dir(e):
			raise
		if e.errno in (errno.EACCES, errno.EAGAIN):
			# resource temp unavailable; eg, someone beat us to the lock.
			if flags & os.O_NONBLOCK:
				raise TryAgain(mypath)

			global _quiet
			out = EOutput()
			out.quiet = _quiet
			if waiting_msg is None:
				if isinstance(mypath, int):
					waiting_msg = _("waiting for lock on fd %i") % myfd
				else:
					waiting_msg = _("waiting for lock on %s\n") % lockfilename
			out.ebegin(waiting_msg)
			# try for the exclusive lock now.
			try:
				fcntl.lockf(myfd, fcntl.LOCK_EX)
			except EnvironmentError as e:
				out.eend(1, str(e))
				raise
			out.eend(os.EX_OK)
		elif e.errno == errno.ENOLCK:
			# We're not allowed to lock on this FS.
			os.close(myfd)
			link_success = False
			if lockfilename == str(lockfilename):
				if wantnewlockfile:
					try:
						if os.stat(lockfilename)[stat.ST_NLINK] == 1:
							os.unlink(lockfilename)
					except OSError:
						pass
					link_success = hardlink_lockfile(lockfilename)
			if not link_success:
				raise
			locking_method = None
			myfd = HARDLINK_FD
		else:
			raise

		
	if isinstance(lockfilename, basestring) and \
		myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
		# The file was deleted on us... Keep trying to make one...
		os.close(myfd)
		writemsg(_("lockfile recurse\n"), 1)
		lockfilename, myfd, unlinkfile, locking_method = lockfile(
			mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
			waiting_msg=waiting_msg, flags=flags)

	writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1)
	return (lockfilename,myfd,unlinkfile,locking_method)
Exemple #47
0
	def _start(self):

		need_builddir = self.phase not in self._phases_without_builddir

		# This can happen if the pre-clean phase triggers
		# die_hooks for some reason, and PORTAGE_BUILDDIR
		# doesn't exist yet.
		if need_builddir and \
			not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
			msg = _("The ebuild phase '%s' has been aborted "
			"since PORTAGE_BUILDDIR does not exist: '%s'") % \
			(self.phase, self.settings['PORTAGE_BUILDDIR'])
			self._eerror(textwrap.wrap(msg, 72))
			self._set_returncode((self.pid, 1 << 8))
			self._async_wait()
			return

		# Check if the cgroup hierarchy is in place. If it's not, mount it.
		if (os.geteuid() == 0 and platform.system() == 'Linux'
				and 'cgroup' in self.settings.features
				and self.phase not in self._phases_without_cgroup):
			cgroup_root = '/sys/fs/cgroup'
			cgroup_portage = os.path.join(cgroup_root, 'portage')

			try:
				# cgroup tmpfs
				if not os.path.ismount(cgroup_root):
					# we expect /sys/fs to be there already
					if not os.path.isdir(cgroup_root):
						os.mkdir(cgroup_root, 0o755)
					subprocess.check_call(['mount', '-t', 'tmpfs',
						'-o', 'rw,nosuid,nodev,noexec,mode=0755',
						'tmpfs', cgroup_root])

				# portage subsystem
				if not os.path.ismount(cgroup_portage):
					if not os.path.isdir(cgroup_portage):
						os.mkdir(cgroup_portage, 0o755)
					subprocess.check_call(['mount', '-t', 'cgroup',
						'-o', 'rw,nosuid,nodev,noexec,none,name=portage',
						'tmpfs', cgroup_portage])

				cgroup_path = tempfile.mkdtemp(dir=cgroup_portage,
					prefix='%s:%s.' % (self.settings["CATEGORY"],
					self.settings["PF"]))
			except (subprocess.CalledProcessError, OSError):
				pass
			else:
				self.cgroup = cgroup_path

		if self.background:
			# Automatically prevent color codes from showing up in logs,
			# since we're not displaying to a terminal anyway.
			self.settings['NOCOLOR'] = 'true'

		if self._enable_ipc_daemon:
			self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
			if self.phase not in self._phases_without_builddir:
				if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings:
					self._build_dir = EbuildBuildDir(
						scheduler=self.scheduler, settings=self.settings)
					self._build_dir.lock()
				self.settings['PORTAGE_IPC_DAEMON'] = "1"
				self._start_ipc_daemon()
			else:
				self.settings.pop('PORTAGE_IPC_DAEMON', None)
		else:
			# Since the IPC daemon is disabled, use a simple tempfile based
			# approach to detect unexpected exit like in bug #190128.
			self.settings.pop('PORTAGE_IPC_DAEMON', None)
			if self.phase not in self._phases_without_builddir:
				exit_file = os.path.join(
					self.settings['PORTAGE_BUILDDIR'],
					'.exit_status')
				self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
				try:
					os.unlink(exit_file)
				except OSError:
					if os.path.exists(exit_file):
						# make sure it doesn't exist
						raise
			else:
				self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)

		if self.fd_pipes is None:
			self.fd_pipes = {}
		null_fd = None
		if 0 not in self.fd_pipes and \
			self.phase not in self._phases_interactive_whitelist and \
			"interactive" not in self.settings.get("PROPERTIES", "").split():
			null_fd = os.open('/dev/null', os.O_RDONLY)
			self.fd_pipes[0] = null_fd

		try:
			SpawnProcess._start(self)
		finally:
			if null_fd is not None:
				os.close(null_fd)
Exemple #48
0
def hardlink_lockfile(lockfilename, max_wait=DeprecationWarning,
	waiting_msg=None, flags=0):
	"""Does the NFS, hardlink shuffle to ensure locking on the disk.
	We create a PRIVATE hardlink to the real lockfile, that is just a
	placeholder on the disk.
	If our file can 2 references, then we have the lock. :)
	Otherwise we lather, rise, and repeat.
	"""

	if max_wait is not DeprecationWarning:
		warnings.warn("The 'max_wait' parameter of "
			"portage.locks.hardlink_lockfile() is now unused. Use "
			"flags=os.O_NONBLOCK instead.",
			DeprecationWarning, stacklevel=2)

	global _quiet
	out = None
	displayed_waiting_msg = False
	preexisting = os.path.exists(lockfilename)
	myhardlock = hardlock_name(lockfilename)

	# Since Python 3.4, chown requires int type (no proxies).
	portage_gid = int(portage.data.portage_gid)

	# myhardlock must not exist prior to our link() call, and we can
	# safely unlink it since its file name is unique to our PID
	try:
		os.unlink(myhardlock)
	except OSError as e:
		if e.errno in (errno.ENOENT, errno.ESTALE):
			pass
		else:
			func_call = "unlink('%s')" % myhardlock
			if e.errno == OperationNotPermitted.errno:
				raise OperationNotPermitted(func_call)
			elif e.errno == PermissionDenied.errno:
				raise PermissionDenied(func_call)
			else:
				raise

	while True:
		# create lockfilename if it doesn't exist yet
		try:
			myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
		except OSError as e:
			func_call = "open('%s')" % lockfilename
			if e.errno == OperationNotPermitted.errno:
				raise OperationNotPermitted(func_call)
			elif e.errno == PermissionDenied.errno:
				raise PermissionDenied(func_call)
			elif e.errno == ReadOnlyFileSystem.errno:
				raise ReadOnlyFileSystem(func_call)
			else:
				raise
		else:
			myfd_st = None
			try:
				myfd_st = os.fstat(myfd)
				if not preexisting:
					# Don't chown the file if it is preexisting, since we
					# want to preserve existing permissions in that case.
					if myfd_st.st_gid != portage_gid:
						os.fchown(myfd, -1, portage_gid)
			except OSError as e:
				if e.errno not in (errno.ENOENT, errno.ESTALE):
					writemsg("%s: fchown('%s', -1, %d)\n" % \
						(e, lockfilename, portage_gid), noiselevel=-1)
					writemsg(_("Cannot chown a lockfile: '%s'\n") % \
						lockfilename, noiselevel=-1)
					writemsg(_("Group IDs of current user: %s\n") % \
						" ".join(str(n) for n in os.getgroups()),
						noiselevel=-1)
				else:
					# another process has removed the file, so we'll have
					# to create it again
					continue
			finally:
				os.close(myfd)

			# If fstat shows more than one hardlink, then it's extremely
			# unlikely that the following link call will result in a lock,
			# so optimize away the wasteful link call and sleep or raise
			# TryAgain.
			if myfd_st is not None and myfd_st.st_nlink < 2:
				try:
					os.link(lockfilename, myhardlock)
				except OSError as e:
					func_call = "link('%s', '%s')" % (lockfilename, myhardlock)
					if e.errno == OperationNotPermitted.errno:
						raise OperationNotPermitted(func_call)
					elif e.errno == PermissionDenied.errno:
						raise PermissionDenied(func_call)
					elif e.errno in (errno.ESTALE, errno.ENOENT):
						# another process has removed the file, so we'll have
						# to create it again
						continue
					else:
						raise
				else:
					if hardlink_is_mine(myhardlock, lockfilename):
						if out is not None:
							out.eend(os.EX_OK)
						break

					try:
						os.unlink(myhardlock)
					except OSError as e:
						# This should not happen, since the file name of
						# myhardlock is unique to our host and PID,
						# and the above link() call succeeded.
						if e.errno not in (errno.ENOENT, errno.ESTALE):
							raise
						raise FileNotFound(myhardlock)

		if flags & os.O_NONBLOCK:
			raise TryAgain(lockfilename)

		if out is None and not _quiet:
			out = portage.output.EOutput()
		if out is not None and not displayed_waiting_msg:
			displayed_waiting_msg = True
			if waiting_msg is None:
				waiting_msg = _("waiting for lock on %s\n") % lockfilename
			out.ebegin(waiting_msg)

		time.sleep(_HARDLINK_POLL_LATENCY)

	return True
Exemple #49
0
	def _start(self):

		if self.fd_pipes is None:
			self.fd_pipes = {}
		else:
			self.fd_pipes = self.fd_pipes.copy()
		fd_pipes = self.fd_pipes

		master_fd, slave_fd = self._pipe(fd_pipes)

		can_log = self._can_log(slave_fd)
		if can_log:
			log_file_path = self.logfile
		else:
			log_file_path = None

		null_input = None
		if not self.background or 0 in fd_pipes:
			# Subclasses such as AbstractEbuildProcess may have already passed
			# in a null file descriptor in fd_pipes, so use that when given.
			pass
		else:
			# TODO: Use job control functions like tcsetpgrp() to control
			# access to stdin. Until then, use /dev/null so that any
			# attempts to read from stdin will immediately return EOF
			# instead of blocking indefinitely.
			null_input = os.open('/dev/null', os.O_RDWR)
			fd_pipes[0] = null_input

		fd_pipes.setdefault(0, portage._get_stdin().fileno())
		fd_pipes.setdefault(1, sys.__stdout__.fileno())
		fd_pipes.setdefault(2, sys.__stderr__.fileno())

		# flush any pending output
		stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
		for fd in fd_pipes.values():
			if fd in stdout_filenos:
				sys.__stdout__.flush()
				sys.__stderr__.flush()
				break

		fd_pipes_orig = fd_pipes.copy()

		if log_file_path is not None or self.background:
			fd_pipes[1] = slave_fd
			fd_pipes[2] = slave_fd

		else:
			# Create a dummy pipe that PipeLogger uses to efficiently
			# monitor for process exit by listening for the EOF event.
			# Re-use of the allocated fd number for the key in fd_pipes
			# guarantees that the keys will not collide for similarly
			# allocated pipes which are used by callers such as
			# FileDigester and MergeProcess. See the _setup_pipes
			# docstring for more benefits of this allocation approach.
			self._dummy_pipe_fd = slave_fd
			fd_pipes[slave_fd] = slave_fd

		kwargs = {}
		for k in self._spawn_kwarg_names:
			v = getattr(self, k)
			if v is not None:
				kwargs[k] = v

		kwargs["fd_pipes"] = fd_pipes
		kwargs["returnpid"] = True
		kwargs.pop("logfile", None)

		retval = self._spawn(self.args, **kwargs)

		os.close(slave_fd)
		if null_input is not None:
			os.close(null_input)

		if isinstance(retval, int):
			# spawn failed
			self._unregister()
			self._set_returncode((self.pid, retval))
			self._async_wait()
			return

		self.pid = retval[0]

		stdout_fd = None
		if can_log and not self.background:
			stdout_fd = os.dup(fd_pipes_orig[1])
			# FD_CLOEXEC is enabled by default in Python >=3.4.
			if sys.hexversion < 0x3040000 and fcntl is not None:
				try:
					fcntl.FD_CLOEXEC
				except AttributeError:
					pass
				else:
					fcntl.fcntl(stdout_fd, fcntl.F_SETFD,
						fcntl.fcntl(stdout_fd,
						fcntl.F_GETFD) | fcntl.FD_CLOEXEC)

		self._pipe_logger = PipeLogger(background=self.background,
			scheduler=self.scheduler, input_fd=master_fd,
			log_file_path=log_file_path,
			stdout_fd=stdout_fd)
		self._pipe_logger.addExitListener(self._pipe_logger_exit)
		self._pipe_logger.start()
		self._registered = True
Exemple #50
0
    def _fetch_uri(self, uri):

        if self.config.options.dry_run:
            # Simply report success.
            logging.info("dry-run: fetch '%s' from '%s'" %
                         (self.distfile, uri))
            self._success()
            self.returncode = os.EX_OK
            self.wait()
            return

        if self.config.options.temp_dir:
            self._fetch_tmp_dir_info = 'temp-dir'
            distdir = self.config.options.temp_dir
        else:
            self._fetch_tmp_dir_info = 'distfiles'
            distdir = self.config.options.distfiles

        tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % os.getpid()

        variables = {"DISTDIR": distdir, "URI": uri, "FILE": tmp_basename}

        self._fetch_tmp_file = os.path.join(distdir, tmp_basename)

        try:
            os.unlink(self._fetch_tmp_file)
        except OSError:
            pass

        args = portage.util.shlex_split(default_fetchcommand)
        args = [portage.util.varexpand(x, mydict=variables) for x in args]

        if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
         not os.path.isabs(args[0]):
            # Python 3.1 _execvp throws TypeError for non-absolute executable
            # path passed as bytes (see http://bugs.python.org/issue8513).
            fullname = portage.process.find_binary(args[0])
            if fullname is None:
                raise portage.exception.CommandNotFound(args[0])
            args[0] = fullname

        args = [
            _unicode_encode(x, encoding=_encodings['fs'], errors='strict')
            for x in args
        ]

        null_fd = os.open(os.devnull, os.O_RDONLY)
        fetcher = PopenProcess(background=self.background,
                               proc=subprocess.Popen(args,
                                                     stdin=null_fd,
                                                     stdout=subprocess.PIPE,
                                                     stderr=subprocess.STDOUT),
                               scheduler=self.scheduler)
        os.close(null_fd)

        fetcher.pipe_reader = PipeLogger(background=self.background,
                                         input_fd=fetcher.proc.stdout,
                                         log_file_path=self._log_path,
                                         scheduler=self.scheduler)

        self._start_task(fetcher, self._fetcher_exit)
Exemple #51
0
def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=None, flags=0):
    """
	If wantnewlockfile is True then this creates a lockfile in the parent
	directory as the file: '.' + basename + '.portage_lockfile'.
	"""

    if not mypath:
        raise InvalidData(_("Empty path given"))

        # Support for file object or integer file descriptor parameters is
        # deprecated due to ambiguity in whether or not it's safe to close
        # the file descriptor, making it prone to "Bad file descriptor" errors
        # or file descriptor leaks.
    if isinstance(mypath, basestring) and mypath[-1] == "/":
        mypath = mypath[:-1]

    lockfilename_path = mypath
    if hasattr(mypath, "fileno"):
        warnings.warn(
            "portage.locks.lockfile() support for " "file object parameters is deprecated. Use a file path instead.",
            DeprecationWarning,
            stacklevel=2,
        )
        lockfilename_path = getattr(mypath, "name", None)
        mypath = mypath.fileno()
    if isinstance(mypath, int):
        warnings.warn(
            "portage.locks.lockfile() support for integer file "
            "descriptor parameters is deprecated. Use a file path instead.",
            DeprecationWarning,
            stacklevel=2,
        )
        lockfilename = mypath
        wantnewlockfile = 0
        unlinkfile = 0
    elif wantnewlockfile:
        base, tail = os.path.split(mypath)
        lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
        lockfilename_path = lockfilename
        unlinkfile = 1
    else:
        lockfilename = mypath

    if isinstance(mypath, basestring):
        if not os.path.exists(os.path.dirname(mypath)):
            raise DirectoryNotFound(os.path.dirname(mypath))
        preexisting = os.path.exists(lockfilename)
        old_mask = os.umask(000)
        try:
            try:
                myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660)
            except OSError as e:
                func_call = "open('%s')" % lockfilename
                if e.errno == OperationNotPermitted.errno:
                    raise OperationNotPermitted(func_call)
                elif e.errno == PermissionDenied.errno:
                    raise PermissionDenied(func_call)
                else:
                    raise

            if not preexisting:
                try:
                    if os.stat(lockfilename).st_gid != portage_gid:
                        os.chown(lockfilename, -1, portage_gid)
                except OSError as e:
                    if e.errno in (errno.ENOENT, errno.ESTALE):
                        return lockfile(
                            mypath,
                            wantnewlockfile=wantnewlockfile,
                            unlinkfile=unlinkfile,
                            waiting_msg=waiting_msg,
                            flags=flags,
                        )
                    else:
                        writemsg("%s: chown('%s', -1, %d)\n" % (e, lockfilename, portage_gid), noiselevel=-1)
                        writemsg(_("Cannot chown a lockfile: '%s'\n") % lockfilename, noiselevel=-1)
                        writemsg(
                            _("Group IDs of current user: %s\n") % " ".join(str(n) for n in os.getgroups()),
                            noiselevel=-1,
                        )
        finally:
            os.umask(old_mask)

    elif isinstance(mypath, int):
        myfd = mypath

    else:
        raise ValueError(_("Unknown type passed in '%s': '%s'") % (type(mypath), mypath))

        # try for a non-blocking lock, if it's held, throw a message
        # we're waiting on lockfile and use a blocking attempt.
    locking_method = _default_lock_fn
    try:
        if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
            raise IOError(errno.ENOSYS, "Function not implemented")
        locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
    except IOError as e:
        if not hasattr(e, "errno"):
            raise
        if e.errno in (errno.EACCES, errno.EAGAIN):
            # resource temp unavailable; eg, someone beat us to the lock.
            if flags & os.O_NONBLOCK:
                os.close(myfd)
                raise TryAgain(mypath)

            global _quiet
            if _quiet:
                out = None
            else:
                out = portage.output.EOutput()
            if waiting_msg is None:
                if isinstance(mypath, int):
                    waiting_msg = _("waiting for lock on fd %i") % myfd
                else:
                    waiting_msg = _("waiting for lock on %s\n") % lockfilename
            if out is not None:
                out.ebegin(waiting_msg)
                # try for the exclusive lock now.
            try:
                locking_method(myfd, fcntl.LOCK_EX)
            except EnvironmentError as e:
                if out is not None:
                    out.eend(1, str(e))
                raise
            if out is not None:
                out.eend(os.EX_OK)
        elif e.errno in (errno.ENOSYS, errno.ENOLCK):
            # We're not allowed to lock on this FS.
            if not isinstance(lockfilename, int):
                # If a file object was passed in, it's not safe
                # to close the file descriptor because it may
                # still be in use.
                os.close(myfd)
            lockfilename_path = _unicode_decode(lockfilename_path, encoding=_encodings["fs"], errors="strict")
            if not isinstance(lockfilename_path, basestring):
                raise
            link_success = hardlink_lockfile(lockfilename_path, waiting_msg=waiting_msg, flags=flags)
            if not link_success:
                raise
            lockfilename = lockfilename_path
            locking_method = None
            myfd = HARDLINK_FD
        else:
            raise

    if isinstance(lockfilename, basestring) and myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
        # The file was deleted on us... Keep trying to make one...
        os.close(myfd)
        writemsg(_("lockfile recurse\n"), 1)
        lockfilename, myfd, unlinkfile, locking_method = lockfile(
            mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile, waiting_msg=waiting_msg, flags=flags
        )

    if myfd != HARDLINK_FD:
        _open_fds.add(myfd)

    writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1)
    return (lockfilename, myfd, unlinkfile, locking_method)