def _parse_color_map(config_root="/", onerror=None): """ Parse /etc/portage/color.map and return a dict of error codes. @param onerror: an optional callback to handle any ParseError that would otherwise be raised @type onerror: callable @rtype: dict @return: a dictionary mapping color classes to color codes """ global codes, _styles myfile = os.path.join(config_root, COLOR_MAP_FILE) ansi_code_pattern = re.compile("^[0-9;]*m$") quotes = "'\"" def strip_quotes(token): if token[0] in quotes and token[0] == token[-1]: token = token[1:-1] return token try: with io.open( _unicode_encode(myfile, encoding=_encodings["fs"], errors="strict"), mode="r", encoding=_encodings["content"], errors="replace", ) as f: lines = f.readlines() for lineno, line in enumerate(lines): commenter_pos = line.find("#") line = line[:commenter_pos].strip() if len(line) == 0: continue split_line = line.split("=") if len(split_line) != 2: e = ParseError( _("'%s', line %s: expected exactly one occurrence of '=' operator" ) % (myfile, lineno)) raise e if onerror: onerror(e) else: raise e continue k = strip_quotes(split_line[0].strip()) v = strip_quotes(split_line[1].strip()) if not k in _styles and not k in codes: e = ParseError( _("'%s', line %s: Unknown variable: '%s'") % (myfile, lineno, k)) if onerror: onerror(e) else: raise e continue if ansi_code_pattern.match(v): if k in _styles: _styles[k] = (esc_seq + v, ) elif k in codes: codes[k] = esc_seq + v else: code_list = [] for x in v.split(): if x in codes: if k in _styles: code_list.append(x) elif k in codes: code_list.append(codes[x]) else: e = ParseError( _("'%s', line %s: Undefined: '%s'") % (myfile, lineno, x)) if onerror: onerror(e) else: raise e if k in _styles: _styles[k] = tuple(code_list) elif k in codes: codes[k] = "".join(code_list) except (IOError, OSError) as e: if e.errno == errno.ENOENT: raise FileNotFound(myfile) elif e.errno == errno.EACCES: raise PermissionDenied(myfile) raise
def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=None, flags=0): """ If wantnewlockfile is True then this creates a lockfile in the parent directory as the file: '.' + basename + '.portage_lockfile'. """ import fcntl if not mypath: raise InvalidData(_("Empty path given")) if isinstance(mypath, basestring) and mypath[-1] == '/': mypath = mypath[:-1] if hasattr(mypath, 'fileno'): mypath = mypath.fileno() if isinstance(mypath, int): lockfilename = mypath wantnewlockfile = 0 unlinkfile = 0 elif wantnewlockfile: base, tail = os.path.split(mypath) lockfilename = os.path.join(base, "." + tail + ".portage_lockfile") del base, tail unlinkfile = 1 else: lockfilename = mypath if isinstance(mypath, basestring): if not os.path.exists(os.path.dirname(mypath)): raise DirectoryNotFound(os.path.dirname(mypath)) preexisting = os.path.exists(lockfilename) old_mask = os.umask(000) try: try: myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660) except OSError as e: func_call = "open('%s')" % lockfilename if e.errno == OperationNotPermitted.errno: raise OperationNotPermitted(func_call) elif e.errno == PermissionDenied.errno: raise PermissionDenied(func_call) else: raise if not preexisting: try: if os.stat(lockfilename).st_gid != portage_gid: os.chown(lockfilename, -1, portage_gid) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE): return lockfile(mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile, waiting_msg=waiting_msg, flags=flags) else: writemsg("%s: chown('%s', -1, %d)\n" % \ (e, lockfilename, portage_gid), noiselevel=-1) writemsg(_("Cannot chown a lockfile: '%s'\n") % \ lockfilename, noiselevel=-1) writemsg(_("Group IDs of current user: %s\n") % \ " ".join(str(n) for n in os.getgroups()), noiselevel=-1) finally: os.umask(old_mask) elif isinstance(mypath, int): myfd = mypath else: raise ValueError(_("Unknown type passed in '%s': '%s'") % \ (type(mypath), mypath)) # try for a non-blocking lock, if it's held, throw a message # we're waiting on lockfile and use a blocking attempt. locking_method = fcntl.lockf try: fcntl.lockf(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: if "errno" not in dir(e): raise if e.errno in (errno.EACCES, errno.EAGAIN): # resource temp unavailable; eg, someone beat us to the lock. if flags & os.O_NONBLOCK: raise TryAgain(mypath) global _quiet out = EOutput() out.quiet = _quiet if waiting_msg is None: if isinstance(mypath, int): waiting_msg = _("waiting for lock on fd %i") % myfd else: waiting_msg = _("waiting for lock on %s\n") % lockfilename out.ebegin(waiting_msg) # try for the exclusive lock now. try: fcntl.lockf(myfd, fcntl.LOCK_EX) except EnvironmentError as e: out.eend(1, str(e)) raise out.eend(os.EX_OK) elif e.errno == errno.ENOLCK: # We're not allowed to lock on this FS. os.close(myfd) link_success = False if lockfilename == str(lockfilename): if wantnewlockfile: try: if os.stat(lockfilename)[stat.ST_NLINK] == 1: os.unlink(lockfilename) except OSError: pass link_success = hardlink_lockfile(lockfilename) if not link_success: raise locking_method = None myfd = HARDLINK_FD else: raise if isinstance(lockfilename, basestring) and \ myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0: # The file was deleted on us... Keep trying to make one... os.close(myfd) writemsg(_("lockfile recurse\n"), 1) lockfilename, myfd, unlinkfile, locking_method = lockfile( mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile, waiting_msg=waiting_msg, flags=flags) writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1) return (lockfilename, myfd, unlinkfile, locking_method)
def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): if isinstance(expand, dict): # Some existing variable definitions have been # passed in, for use in substitutions. expand_map = expand expand = True else: expand_map = {} mykeys = {} f = None try: # NOTE: shlex doesn't support unicode objects with Python 2 # (produces spurious \0 characters). if sys.hexversion < 0x3000000: f = open(_unicode_encode(mycfg, encoding=_encodings['fs'], errors='strict'), 'rb') else: f = open(_unicode_encode(mycfg, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='replace') content = f.read() except IOError as e: if e.errno == PermissionDenied.errno: raise PermissionDenied(mycfg) if e.errno != errno.ENOENT: writemsg("open('%s', 'r'): %s\n" % (mycfg, e), noiselevel=-1) if e.errno not in (errno.EISDIR,): raise return None finally: if f is not None: f.close() # Workaround for avoiding a silent error in shlex that is # triggered by a source statement at the end of the file # without a trailing newline after the source statement. if content and content[-1] != '\n': content += '\n' # Warn about dos-style line endings since that prevents # people from being able to source them with bash. if '\r' in content: writemsg(("!!! " + _("Please use dos2unix to convert line endings " + \ "in config file: '%s'") + "\n") % mycfg, noiselevel=-1) lex = None try: if tolerant: shlex_class = _tolerant_shlex else: shlex_class = shlex.shlex # The default shlex.sourcehook() implementation # only joins relative paths when the infile # attribute is properly set. lex = shlex_class(content, infile=mycfg, posix=True) lex.wordchars = string.digits + string.ascii_letters + \ "~!@#$%*_\:;?,./-+{}" lex.quotes="\"'" if allow_sourcing: lex.source="source" while 1: key=lex.get_token() if key == "export": key = lex.get_token() if key is None: #normal end of file break; equ=lex.get_token() if (equ==''): msg = lex.error_leader() + _("Unexpected EOF") if not tolerant: raise ParseError(msg) else: writemsg("%s\n" % msg, noiselevel=-1) return mykeys elif (equ!='='): msg = lex.error_leader() + \ _("Invalid token '%s' (not '=')") % (equ,) if not tolerant: raise ParseError(msg) else: writemsg("%s\n" % msg, noiselevel=-1) return mykeys val=lex.get_token() if val is None: msg = lex.error_leader() + \ _("Unexpected end of config file: variable '%s'") % (key,) if not tolerant: raise ParseError(msg) else: writemsg("%s\n" % msg, noiselevel=-1) return mykeys key = _unicode_decode(key) val = _unicode_decode(val) if _invalid_var_name_re.search(key) is not None: msg = lex.error_leader() + \ _("Invalid variable name '%s'") % (key,) if not tolerant: raise ParseError(msg) writemsg("%s\n" % msg, noiselevel=-1) continue if expand: mykeys[key] = varexpand(val, mydict=expand_map, error_leader=lex.error_leader) expand_map[key] = mykeys[key] else: mykeys[key] = val except SystemExit as e: raise except Exception as e: if isinstance(e, ParseError) or lex is None: raise msg = _unicode_decode("%s%s") % (lex.error_leader(), e) writemsg("%s\n" % msg, noiselevel=-1) raise return mykeys
def apply_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1, stat_cached=None, follow_links=True): """Apply user, group, and mode bits to a file if the existing bits do not already match. The default behavior is to force an exact match of mode bits. When mask=0 is specified, mode bits on the target file are allowed to be a superset of the mode argument (via logical OR). When mask>0, the mode bits that the target file is allowed to have are restricted via logical XOR. Returns True if the permissions were modified and False otherwise.""" modified = False if stat_cached is None: try: if follow_links: stat_cached = os.stat(filename) else: stat_cached = os.lstat(filename) except OSError as oe: func_call = "stat('%s')" % filename if oe.errno == errno.EPERM: raise OperationNotPermitted(func_call) elif oe.errno == errno.EACCES: raise PermissionDenied(func_call) elif oe.errno == errno.ENOENT: raise FileNotFound(filename) else: raise if (uid != -1 and uid != stat_cached.st_uid) or \ (gid != -1 and gid != stat_cached.st_gid): try: if follow_links: os.chown(filename, uid, gid) else: portage.data.lchown(filename, uid, gid) modified = True except OSError as oe: func_call = "chown('%s', %i, %i)" % (filename, uid, gid) if oe.errno == errno.EPERM: raise OperationNotPermitted(func_call) elif oe.errno == errno.EACCES: raise PermissionDenied(func_call) elif oe.errno == errno.EROFS: raise ReadOnlyFileSystem(func_call) elif oe.errno == errno.ENOENT: raise FileNotFound(filename) else: raise new_mode = -1 st_mode = stat_cached.st_mode & 0o7777 # protect from unwanted bits if mask >= 0: if mode == -1: mode = 0 # Don't add any mode bits when mode is unspecified. else: mode = mode & 0o7777 if (mode & st_mode != mode) or \ ((mask ^ st_mode) & st_mode != st_mode): new_mode = mode | st_mode new_mode = (mask ^ new_mode) & new_mode elif mode != -1: mode = mode & 0o7777 # protect from unwanted bits if mode != st_mode: new_mode = mode # The chown system call may clear S_ISUID and S_ISGID # bits, so those bits are restored if necessary. if modified and new_mode == -1 and \ (st_mode & stat.S_ISUID or st_mode & stat.S_ISGID): if mode == -1: new_mode = st_mode else: mode = mode & 0o7777 if mask >= 0: new_mode = mode | st_mode new_mode = (mask ^ new_mode) & new_mode else: new_mode = mode if not (new_mode & stat.S_ISUID or new_mode & stat.S_ISGID): new_mode = -1 if not follow_links and stat.S_ISLNK(stat_cached.st_mode): # Mode doesn't matter for symlinks. new_mode = -1 if new_mode != -1: try: os.chmod(filename, new_mode) modified = True except OSError as oe: func_call = "chmod('%s', %s)" % (filename, oct(new_mode)) if oe.errno == errno.EPERM: raise OperationNotPermitted(func_call) elif oe.errno == errno.EACCES: raise PermissionDenied(func_call) elif oe.errno == errno.EROFS: raise ReadOnlyFileSystem(func_call) elif oe.errno == errno.ENOENT: raise FileNotFound(filename) raise return modified
def hardlink_lockfile(lockfilename, max_wait=DeprecationWarning, waiting_msg=None, flags=0): """Does the NFS, hardlink shuffle to ensure locking on the disk. We create a PRIVATE hardlink to the real lockfile, that is just a placeholder on the disk. If our file can 2 references, then we have the lock. :) Otherwise we lather, rise, and repeat. """ if max_wait is not DeprecationWarning: warnings.warn( "The 'max_wait' parameter of " "portage.locks.hardlink_lockfile() is now unused. Use " "flags=os.O_NONBLOCK instead.", DeprecationWarning, stacklevel=2) global _quiet out = None displayed_waiting_msg = False preexisting = os.path.exists(lockfilename) myhardlock = hardlock_name(lockfilename) # Since Python 3.4, chown requires int type (no proxies). portage_gid = int(portage.data.portage_gid) # myhardlock must not exist prior to our link() call, and we can # safely unlink it since its file name is unique to our PID try: os.unlink(myhardlock) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE): pass else: func_call = "unlink('%s')" % myhardlock if e.errno == OperationNotPermitted.errno: raise OperationNotPermitted(func_call) elif e.errno == PermissionDenied.errno: raise PermissionDenied(func_call) else: raise while True: # create lockfilename if it doesn't exist yet try: myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660) except OSError as e: func_call = "open('%s')" % lockfilename if e.errno == OperationNotPermitted.errno: raise OperationNotPermitted(func_call) elif e.errno == PermissionDenied.errno: raise PermissionDenied(func_call) elif e.errno == ReadOnlyFileSystem.errno: raise ReadOnlyFileSystem(func_call) else: raise else: myfd_st = None try: myfd_st = os.fstat(myfd) if not preexisting: # Don't chown the file if it is preexisting, since we # want to preserve existing permissions in that case. if portage.data.secpass >= 1 and myfd_st.st_gid != portage_gid: os.fchown(myfd, -1, portage_gid) except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): writemsg("%s: fchown('%s', -1, %d)\n" % \ (e, lockfilename, portage_gid), noiselevel=-1) writemsg(_("Cannot chown a lockfile: '%s'\n") % \ lockfilename, noiselevel=-1) writemsg(_("Group IDs of current user: %s\n") % \ " ".join(str(n) for n in os.getgroups()), noiselevel=-1) else: # another process has removed the file, so we'll have # to create it again continue finally: os.close(myfd) # If fstat shows more than one hardlink, then it's extremely # unlikely that the following link call will result in a lock, # so optimize away the wasteful link call and sleep or raise # TryAgain. if myfd_st is not None and myfd_st.st_nlink < 2: try: os.link(lockfilename, myhardlock) except OSError as e: func_call = "link('%s', '%s')" % (lockfilename, myhardlock) if e.errno == OperationNotPermitted.errno: raise OperationNotPermitted(func_call) elif e.errno == PermissionDenied.errno: raise PermissionDenied(func_call) elif e.errno in (errno.ESTALE, errno.ENOENT): # another process has removed the file, so we'll have # to create it again continue else: raise else: if hardlink_is_mine(myhardlock, lockfilename): if out is not None: out.eend(os.EX_OK) break try: os.unlink(myhardlock) except OSError as e: # This should not happen, since the file name of # myhardlock is unique to our host and PID, # and the above link() call succeeded. if e.errno not in (errno.ENOENT, errno.ESTALE): raise raise FileNotFound(myhardlock) if flags & os.O_NONBLOCK: raise TryAgain(lockfilename) if out is None and not _quiet: out = portage.output.EOutput() if out is not None and not displayed_waiting_msg: displayed_waiting_msg = True if waiting_msg is None: waiting_msg = _("waiting for lock on %s\n") % lockfilename out.ebegin(waiting_msg) time.sleep(_HARDLINK_POLL_LATENCY) return True
def write(self, sign=False, force=False): """ Write Manifest instance to disk, optionally signing it. Returns True if the Manifest is actually written, and False if the write is skipped due to existing Manifest being identical.""" rval = False if not self.allow_create: return rval self.checkIntegrity() try: myentries = list(self._createManifestEntries()) update_manifest = True preserved_stats = {} preserved_stats[self.pkgdir.rstrip(os.sep)] = os.stat(self.pkgdir) if myentries and not force: try: f = io.open(_unicode_encode(self.getFullname(), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') oldentries = list(self._parseManifestLines(f)) preserved_stats[self.getFullname()] = os.fstat(f.fileno()) f.close() if len(oldentries) == len(myentries): update_manifest = False for i in range(len(oldentries)): if oldentries[i] != myentries[i]: update_manifest = True break except (IOError, OSError) as e: if e.errno == errno.ENOENT: pass else: raise if update_manifest: if myentries or not (self.thin or self.allow_missing): # If myentries is empty, don't write an empty manifest # when thin or allow_missing is enabled. Except for # thin manifests with no DIST entries, myentries is # non-empty for all currently known use cases. write_atomic(self.getFullname(), "".join("%s\n" % _unicode(myentry) for myentry in myentries)) self._apply_max_mtime(preserved_stats, myentries) rval = True else: # With thin manifest, there's no need to have # a Manifest file if there are no DIST entries. try: os.unlink(self.getFullname()) except OSError as e: if e.errno != errno.ENOENT: raise rval = True if sign: self.sign() except (IOError, OSError) as e: if e.errno == errno.EACCES: raise PermissionDenied(str(e)) raise return rval
def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=None, flags=0): """ If wantnewlockfile is True then this creates a lockfile in the parent directory as the file: '.' + basename + '.portage_lockfile'. """ if not mypath: raise InvalidData(_("Empty path given")) # Since Python 3.4, chown requires int type (no proxies). portage_gid = int(portage.data.portage_gid) # Support for file object or integer file descriptor parameters is # deprecated due to ambiguity in whether or not it's safe to close # the file descriptor, making it prone to "Bad file descriptor" errors # or file descriptor leaks. if isinstance(mypath, basestring) and mypath[-1] == '/': mypath = mypath[:-1] lockfilename_path = mypath if hasattr(mypath, 'fileno'): warnings.warn( "portage.locks.lockfile() support for " "file object parameters is deprecated. Use a file path instead.", DeprecationWarning, stacklevel=2) lockfilename_path = getattr(mypath, 'name', None) mypath = mypath.fileno() if isinstance(mypath, int): warnings.warn( "portage.locks.lockfile() support for integer file " "descriptor parameters is deprecated. Use a file path instead.", DeprecationWarning, stacklevel=2) lockfilename = mypath wantnewlockfile = 0 unlinkfile = 0 elif wantnewlockfile: base, tail = os.path.split(mypath) lockfilename = os.path.join(base, "." + tail + ".portage_lockfile") lockfilename_path = lockfilename unlinkfile = 1 else: lockfilename = mypath if isinstance(mypath, basestring): if not os.path.exists(os.path.dirname(mypath)): raise DirectoryNotFound(os.path.dirname(mypath)) preexisting = os.path.exists(lockfilename) old_mask = os.umask(000) try: try: myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660) except OSError as e: func_call = "open('%s')" % lockfilename if e.errno == OperationNotPermitted.errno: raise OperationNotPermitted(func_call) elif e.errno == PermissionDenied.errno: raise PermissionDenied(func_call) elif e.errno == ReadOnlyFileSystem.errno: raise ReadOnlyFileSystem(func_call) else: raise if not preexisting: try: if os.stat(lockfilename).st_gid != portage_gid: os.chown(lockfilename, -1, portage_gid) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE): return lockfile(mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile, waiting_msg=waiting_msg, flags=flags) else: writemsg("%s: chown('%s', -1, %d)\n" % \ (e, lockfilename, portage_gid), noiselevel=-1) writemsg(_("Cannot chown a lockfile: '%s'\n") % \ lockfilename, noiselevel=-1) writemsg(_("Group IDs of current user: %s\n") % \ " ".join(str(n) for n in os.getgroups()), noiselevel=-1) finally: os.umask(old_mask) elif isinstance(mypath, int): myfd = mypath else: raise ValueError(_("Unknown type passed in '%s': '%s'") % \ (type(mypath), mypath)) # try for a non-blocking lock, if it's held, throw a message # we're waiting on lockfile and use a blocking attempt. locking_method = portage._eintr_func_wrapper(_get_lock_fn()) try: if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: raise IOError(errno.ENOSYS, "Function not implemented") locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: if not hasattr(e, "errno"): raise if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK): # resource temp unavailable; eg, someone beat us to the lock. if flags & os.O_NONBLOCK: os.close(myfd) raise TryAgain(mypath) global _quiet if _quiet: out = None else: out = portage.output.EOutput() if waiting_msg is None: if isinstance(mypath, int): waiting_msg = _("waiting for lock on fd %i") % myfd else: waiting_msg = _("waiting for lock on %s") % lockfilename if out is not None: out.ebegin(waiting_msg) # try for the exclusive lock now. enolock_msg_shown = False while True: try: locking_method(myfd, fcntl.LOCK_EX) except EnvironmentError as e: if e.errno == errno.ENOLCK: # This is known to occur on Solaris NFS (see # bug #462694). Assume that the error is due # to temporary exhaustion of record locks, # and loop until one becomes available. if not enolock_msg_shown: enolock_msg_shown = True if isinstance(mypath, int): context_desc = _("Error while waiting " "to lock fd %i") % myfd else: context_desc = _("Error while waiting " "to lock '%s'") % lockfilename writemsg("\n!!! %s: %s\n" % (context_desc, e), noiselevel=-1) time.sleep(_HARDLINK_POLL_LATENCY) continue if out is not None: out.eend(1, str(e)) raise else: break if out is not None: out.eend(os.EX_OK) elif e.errno in (errno.ENOSYS, ): # We're not allowed to lock on this FS. if not isinstance(lockfilename, int): # If a file object was passed in, it's not safe # to close the file descriptor because it may # still be in use. os.close(myfd) lockfilename_path = _unicode_decode(lockfilename_path, encoding=_encodings['fs'], errors='strict') if not isinstance(lockfilename_path, basestring): raise link_success = hardlink_lockfile(lockfilename_path, waiting_msg=waiting_msg, flags=flags) if not link_success: raise lockfilename = lockfilename_path locking_method = None myfd = HARDLINK_FD else: raise if isinstance(lockfilename, basestring) and \ myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0: # The file was deleted on us... Keep trying to make one... os.close(myfd) writemsg(_("lockfile recurse\n"), 1) lockfilename, myfd, unlinkfile, locking_method = lockfile( mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile, waiting_msg=waiting_msg, flags=flags) if myfd != HARDLINK_FD: # FD_CLOEXEC is enabled by default in Python >=3.4. if sys.hexversion < 0x3040000: try: fcntl.FD_CLOEXEC except AttributeError: pass else: fcntl.fcntl( myfd, fcntl.F_SETFD, fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) _open_fds.add(myfd) writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1) return (lockfilename, myfd, unlinkfile, locking_method)
def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): if isinstance(expand, dict): # Some existing variable definitions have been # passed in, for use in substitutions. expand_map = expand expand = True else: expand_map = {} mykeys = {} try: # Workaround for avoiding a silent error in shlex that # is triggered by a source statement at the end of the file without a # trailing newline after the source statement # NOTE: shex doesn't seem to support unicode objects # (produces spurious \0 characters with python-2.6.2) if sys.hexversion < 0x3000000: content = open(_unicode_encode(mycfg, encoding=_encodings['fs'], errors='strict'), 'rb').read() else: content = open(_unicode_encode(mycfg, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='replace').read() if content and content[-1] != '\n': content += '\n' except IOError as e: if e.errno == PermissionDenied.errno: raise PermissionDenied(mycfg) if e.errno != errno.ENOENT: writemsg("open('%s', 'r'): %s\n" % (mycfg, e), noiselevel=-1) if e.errno not in (errno.EISDIR,): raise return None try: if tolerant: shlex_class = _tolerant_shlex else: shlex_class = shlex.shlex # The default shlex.sourcehook() implementation # only joins relative paths when the infile # attribute is properly set. lex = shlex_class(content, infile=mycfg, posix=True) lex.wordchars = string.digits + string.ascii_letters + \ "~!@#$%*_\:;?,./-+{}" lex.quotes="\"'" if allow_sourcing: lex.source="source" while 1: key=lex.get_token() if key == "export": key = lex.get_token() if key is None: #normal end of file break; equ=lex.get_token() if (equ==''): #unexpected end of file #lex.error_leader(self.filename,lex.lineno) if not tolerant: writemsg(_("!!! Unexpected end of config file: variable %s\n") % key, noiselevel=-1) raise Exception(_("ParseError: Unexpected EOF: %s: on/before line %s") % (mycfg, lex.lineno)) else: return mykeys elif (equ!='='): #invalid token #lex.error_leader(self.filename,lex.lineno) if not tolerant: raise Exception(_("ParseError: Invalid token " "'%s' (not '='): %s: line %s") % \ (equ, mycfg, lex.lineno)) else: return mykeys val=lex.get_token() if val is None: #unexpected end of file #lex.error_leader(self.filename,lex.lineno) if not tolerant: writemsg(_("!!! Unexpected end of config file: variable %s\n") % key, noiselevel=-1) raise portage.exception.CorruptionError(_("ParseError: Unexpected EOF: %s: line %s") % (mycfg, lex.lineno)) else: return mykeys key = _unicode_decode(key) val = _unicode_decode(val) if _invalid_var_name_re.search(key) is not None: if not tolerant: raise Exception(_( "ParseError: Invalid variable name '%s': line %s") % \ (key, lex.lineno - 1)) writemsg(_("!!! Invalid variable name '%s': line %s in %s\n") \ % (key, lex.lineno - 1, mycfg), noiselevel=-1) continue if expand: mykeys[key] = varexpand(val, expand_map) expand_map[key] = mykeys[key] else: mykeys[key] = val except SystemExit as e: raise except Exception as e: raise portage.exception.ParseError(str(e)+" in "+mycfg) return mykeys
def cacheddir(my_original_path, ignorecvs, ignorelist, EmptyOnError, followSymlinks=True): global cacheHit, cacheMiss, cacheStale mypath = normalize_path(my_original_path) if mypath in dircache: cacheHit += 1 cached_mtime, list, ftype = dircache[mypath] else: cacheMiss += 1 cached_mtime, list, ftype = -1, [], [] try: pathstat = os.stat(mypath) if stat.S_ISDIR(pathstat[stat.ST_MODE]): mtime = pathstat.st_mtime else: raise DirectoryNotFound(mypath) except EnvironmentError as e: if e.errno == PermissionDenied.errno: raise PermissionDenied(mypath) del e return [], [] except PortageException: return [], [] # Python retuns mtime in seconds, so if it was changed in the last few seconds, it could be invalid if mtime != cached_mtime or time.time() - mtime < 4: if mypath in dircache: cacheStale += 1 try: list = os.listdir(mypath) except EnvironmentError as e: if e.errno != errno.EACCES: raise del e raise PermissionDenied(mypath) ftype = [] for x in list: try: if followSymlinks: pathstat = os.stat(mypath + "/" + x) else: pathstat = os.lstat(mypath + "/" + x) if stat.S_ISREG(pathstat[stat.ST_MODE]): ftype.append(0) elif stat.S_ISDIR(pathstat[stat.ST_MODE]): ftype.append(1) elif stat.S_ISLNK(pathstat[stat.ST_MODE]): ftype.append(2) else: ftype.append(3) except (IOError, OSError): ftype.append(3) dircache[mypath] = mtime, list, ftype ret_list = [] ret_ftype = [] for x in range(0, len(list)): if list[x] in ignorelist: pass elif ignorecvs: if list[x][:2] != ".#" and \ not (ftype[x] == 1 and list[x] in _ignorecvs_dirs): ret_list.append(list[x]) ret_ftype.append(ftype[x]) else: ret_list.append(list[x]) ret_ftype.append(ftype[x]) writemsg( "cacheddirStats: H:%d/M:%d/S:%d\n" % (cacheHit, cacheMiss, cacheStale), 10) return ret_list, ret_ftype