def builddir_unlocked(future): if future.exception() is not None: result.set_exception(future.exception()) else: self._lock_obj = None self.locked = False self.settings.pop('PORTAGE_BUILDDIR_LOCKED', None) catdir_lock = AsynchronousLock(path=self._catdir, scheduler=self.scheduler) catdir_lock.addExitListener(catdir_locked) catdir_lock.start()
class _BinpkgFetcherProcess(SpawnProcess): __slots__ = ("pkg", "pretend", "locked", "pkg_path", "_lock_obj") def _start(self): pkg = self.pkg pretend = self.pretend bintree = pkg.root_config.trees["bintree"] settings = bintree.settings pkg_path = self.pkg_path exists = os.path.exists(pkg_path) resume = exists and os.path.basename(pkg_path) in bintree.invalids if not (pretend or resume): # Remove existing file or broken symlink. try: os.unlink(pkg_path) except OSError: pass # urljoin doesn't work correctly with # unrecognized protocols like sftp fetchcommand = None resumecommand = None if bintree._remote_has_index: remote_metadata = bintree._remotepkgs[bintree.dbapi._instance_key( pkg.cpv)] rel_uri = remote_metadata.get("PATH") if not rel_uri: rel_uri = pkg.cpv + ".tbz2" remote_base_uri = remote_metadata["BASE_URI"] uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/") fetchcommand = remote_metadata.get('FETCHCOMMAND') resumecommand = remote_metadata.get('RESUMECOMMAND') else: uri = settings["PORTAGE_BINHOST"].rstrip("/") + \ "/" + pkg.pf + ".tbz2" if pretend: portage.writemsg_stdout("\n%s\n" % uri, noiselevel=-1) self.returncode = os.EX_OK self._async_wait() return fcmd = None if resume: fcmd = resumecommand else: fcmd = fetchcommand if fcmd is None: protocol = urllib_parse_urlparse(uri)[0] fcmd_prefix = "FETCHCOMMAND" if resume: fcmd_prefix = "RESUMECOMMAND" fcmd = settings.get(fcmd_prefix + "_" + protocol.upper()) if not fcmd: fcmd = settings.get(fcmd_prefix) fcmd_vars = { "DISTDIR": os.path.dirname(pkg_path), "URI": uri, "FILE": os.path.basename(pkg_path) } for k in ("PORTAGE_SSH_OPTS", ): v = settings.get(k) if v is not None: fcmd_vars[k] = v fetch_env = dict(settings.items()) fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \ for x in portage.util.shlex_split(fcmd)] if self.fd_pipes is None: self.fd_pipes = {} fd_pipes = self.fd_pipes # Redirect all output to stdout since some fetchers like # wget pollute stderr (if portage detects a problem then it # can send it's own message to stderr). fd_pipes.setdefault(0, portage._get_stdin().fileno()) fd_pipes.setdefault(1, sys.__stdout__.fileno()) fd_pipes.setdefault(2, sys.__stdout__.fileno()) self.args = fetch_args self.env = fetch_env if settings.selinux_enabled(): self._selinux_type = settings["PORTAGE_FETCH_T"] self.log_filter_file = settings.get('PORTAGE_LOG_FILTER_FILE_CMD') SpawnProcess._start(self) def _pipe(self, fd_pipes): """When appropriate, use a pty so that fetcher progress bars, like wget has, will work properly.""" if self.background or not sys.__stdout__.isatty(): # When the output only goes to a log file, # there's no point in creating a pty. return os.pipe() stdout_pipe = None if not self.background: stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = \ _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) def sync_timestamp(self): # If possible, update the mtime to match the remote package if # the fetcher didn't already do it automatically. bintree = self.pkg.root_config.trees["bintree"] if bintree._remote_has_index: remote_mtime = bintree._remotepkgs[bintree.dbapi._instance_key( self.pkg.cpv)].get("_mtime_") if remote_mtime is not None: try: remote_mtime = int(remote_mtime) except ValueError: pass else: try: local_mtime = os.stat(self.pkg_path)[stat.ST_MTIME] except OSError: pass else: if remote_mtime != local_mtime: try: os.utime(self.pkg_path, (remote_mtime, remote_mtime)) except OSError: pass def async_lock(self): """ This raises an AlreadyLocked exception if lock() is called while a lock is already held. In order to avoid this, call unlock() or check whether the "locked" attribute is True or False before calling lock(). """ if self._lock_obj is not None: raise self.AlreadyLocked((self._lock_obj, )) result = self.scheduler.create_future() def acquired_lock(async_lock): if async_lock.wait() == os.EX_OK: self.locked = True result.set_result(None) else: result.set_exception( AssertionError( "AsynchronousLock failed with returncode %s" % (async_lock.returncode, ))) self._lock_obj = AsynchronousLock(path=self.pkg_path, scheduler=self.scheduler) self._lock_obj.addExitListener(acquired_lock) self._lock_obj.start() return result class AlreadyLocked(portage.exception.PortageException): pass def async_unlock(self): if self._lock_obj is None: raise AssertionError('already unlocked') result = self._lock_obj.async_unlock() self._lock_obj = None self.locked = False return result
def async_lock(self): """ Acquire the lock asynchronously. Notification is available via the add_done_callback method of the returned Future instance. This raises an AlreadyLocked exception if async_lock() is called while a lock is already held. In order to avoid this, call async_unlock() or check whether the "locked" attribute is True or False before calling async_lock(). @returns: Future, result is None """ if self._lock_obj is not None: raise self.AlreadyLocked((self._lock_obj, )) dir_path = self.settings.get('PORTAGE_BUILDDIR') if not dir_path: raise AssertionError('PORTAGE_BUILDDIR is unset') catdir = os.path.dirname(dir_path) self._catdir = catdir catdir_lock = AsynchronousLock(path=catdir, scheduler=self.scheduler) builddir_lock = AsynchronousLock(path=dir_path, scheduler=self.scheduler) result = self.scheduler.create_future() def catdir_locked(catdir_lock): try: self._assert_lock(catdir_lock) except AssertionError as e: result.set_exception(e) return try: portage.util.ensure_dirs(catdir, gid=portage.portage_gid, mode=0o70, mask=0) except PortageException as e: if not os.path.isdir(catdir): result.set_exception(e) return builddir_lock.addExitListener(builddir_locked) builddir_lock.start() def builddir_locked(builddir_lock): try: self._assert_lock(builddir_lock) except AssertionError as e: catdir_lock.async_unlock.add_done_callback( functools.partial(catdir_unlocked, exception=e)) return self._lock_obj = builddir_lock self.locked = True self.settings['PORTAGE_BUILDDIR_LOCKED'] = '1' catdir_lock.async_unlock().add_done_callback(catdir_unlocked) def catdir_unlocked(future, exception=None): if not (exception is None and future.exception() is None): result.set_exception(exception or future.exception()) else: result.set_result(None) try: portage.util.ensure_dirs(os.path.dirname(catdir), gid=portage.portage_gid, mode=0o70, mask=0) except PortageException: if not os.path.isdir(os.path.dirname(catdir)): raise catdir_lock.addExitListener(catdir_locked) catdir_lock.start() return result
class _BinpkgFetcherProcess(SpawnProcess): __slots__ = ("pkg", "pretend", "locked", "pkg_path", "_lock_obj") def _start(self): pkg = self.pkg pretend = self.pretend bintree = pkg.root_config.trees["bintree"] settings = bintree.settings pkg_path = self.pkg_path exists = os.path.exists(pkg_path) resume = exists and os.path.basename(pkg_path) in bintree.invalids if not (pretend or resume): # Remove existing file or broken symlink. try: os.unlink(pkg_path) except OSError: pass # urljoin doesn't work correctly with # unrecognized protocols like sftp if bintree._remote_has_index: instance_key = bintree.dbapi._instance_key(pkg.cpv) rel_uri = bintree._remotepkgs[instance_key].get("PATH") if not rel_uri: rel_uri = pkg.cpv + ".tbz2" remote_base_uri = bintree._remotepkgs[ instance_key]["BASE_URI"] uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/") else: uri = settings["PORTAGE_BINHOST"].rstrip("/") + \ "/" + pkg.pf + ".tbz2" if pretend: portage.writemsg_stdout("\n%s\n" % uri, noiselevel=-1) self.returncode = os.EX_OK self._async_wait() return protocol = urllib_parse_urlparse(uri)[0] fcmd_prefix = "FETCHCOMMAND" if resume: fcmd_prefix = "RESUMECOMMAND" fcmd = settings.get(fcmd_prefix + "_" + protocol.upper()) if not fcmd: fcmd = settings.get(fcmd_prefix) fcmd_vars = { "DISTDIR" : os.path.dirname(pkg_path), "URI" : uri, "FILE" : os.path.basename(pkg_path) } for k in ("PORTAGE_SSH_OPTS",): v = settings.get(k) if v is not None: fcmd_vars[k] = v fetch_env = dict(settings.items()) fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \ for x in portage.util.shlex_split(fcmd)] if self.fd_pipes is None: self.fd_pipes = {} fd_pipes = self.fd_pipes # Redirect all output to stdout since some fetchers like # wget pollute stderr (if portage detects a problem then it # can send it's own message to stderr). fd_pipes.setdefault(0, portage._get_stdin().fileno()) fd_pipes.setdefault(1, sys.__stdout__.fileno()) fd_pipes.setdefault(2, sys.__stdout__.fileno()) self.args = fetch_args self.env = fetch_env if settings.selinux_enabled(): self._selinux_type = settings["PORTAGE_FETCH_T"] SpawnProcess._start(self) def _pipe(self, fd_pipes): """When appropriate, use a pty so that fetcher progress bars, like wget has, will work properly.""" if self.background or not sys.__stdout__.isatty(): # When the output only goes to a log file, # there's no point in creating a pty. return os.pipe() stdout_pipe = None if not self.background: stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = \ _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) def sync_timestamp(self): # If possible, update the mtime to match the remote package if # the fetcher didn't already do it automatically. bintree = self.pkg.root_config.trees["bintree"] if bintree._remote_has_index: remote_mtime = bintree._remotepkgs[ bintree.dbapi._instance_key( self.pkg.cpv)].get("_mtime_") if remote_mtime is not None: try: remote_mtime = long(remote_mtime) except ValueError: pass else: try: local_mtime = os.stat(self.pkg_path)[stat.ST_MTIME] except OSError: pass else: if remote_mtime != local_mtime: try: os.utime(self.pkg_path, (remote_mtime, remote_mtime)) except OSError: pass def async_lock(self): """ This raises an AlreadyLocked exception if lock() is called while a lock is already held. In order to avoid this, call unlock() or check whether the "locked" attribute is True or False before calling lock(). """ if self._lock_obj is not None: raise self.AlreadyLocked((self._lock_obj,)) result = self.scheduler.create_future() def acquired_lock(async_lock): if async_lock.wait() == os.EX_OK: self.locked = True result.set_result(None) else: result.set_exception(AssertionError( "AsynchronousLock failed with returncode %s" % (async_lock.returncode,))) self._lock_obj = AsynchronousLock(path=self.pkg_path, scheduler=self.scheduler) self._lock_obj.addExitListener(acquired_lock) self._lock_obj.start() return result class AlreadyLocked(portage.exception.PortageException): pass def async_unlock(self): if self._lock_obj is None: raise AssertionError('already unlocked') result = self._lock_obj.async_unlock() self._lock_obj = None self.locked = False return result