def testCompileModules(self):
for parent, dirs, files in itertools.chain(
os.walk(PORTAGE_BIN_PATH),
os.walk(PORTAGE_PYM_PATH)):
parent = _unicode_decode(parent,
encoding=_encodings['fs'], errors='strict')
for x in files:
x = _unicode_decode(x,
encoding=_encodings['fs'], errors='strict')
if x[-4:] in ('.pyc', '.pyo'):
continue
x = os.path.join(parent, x)
st = os.lstat(x)
if not stat.S_ISREG(st.st_mode):
continue
do_compile = False
if x[-3:] == '.py':
do_compile = True
else:
# Check for python shebang
with open(_unicode_encode(x,
encoding=_encodings['fs'], errors='strict'), 'rb') as f:
line = _unicode_decode(f.readline(),
encoding=_encodings['content'], errors='replace')
if line[:2] == '#!' and 'python' in line:
do_compile = True
if do_compile:
with open(_unicode_encode(x,
encoding=_encodings['fs'], errors='strict'), 'rb') as f:
compile(f.read(), x, 'exec')
def __init__(self, filename, mode='w', follow_links=True, **kargs):
"""Opens a temporary filename.pid in the same directory as filename."""
ObjectProxy.__init__(self)
object.__setattr__(self, '_aborted', False)
if 'b' in mode:
open_func = open
else:
open_func = io.open
kargs.setdefault('encoding', _encodings['content'])
kargs.setdefault('errors', 'backslashreplace')
if follow_links:
canonical_path = os.path.realpath(filename)
object.__setattr__(self, '_real_name', canonical_path)
tmp_name = "%s.%i" % (canonical_path, os.getpid())
try:
object.__setattr__(self, '_file',
open_func(_unicode_encode(tmp_name,
encoding=_encodings['fs'], errors='strict'),
mode=mode, **kargs))
return
except IOError as e:
if canonical_path == filename:
raise
# Ignore this error, since it's irrelevant
# and the below open call will produce a
# new error if necessary.
object.__setattr__(self, '_real_name', filename)
tmp_name = "%s.%i" % (filename, os.getpid())
object.__setattr__(self, '_file',
open_func(_unicode_encode(tmp_name,
encoding=_encodings['fs'], errors='strict'),
mode=mode, **kargs))
def update(self):
''' Update existing git repository, and ignore the syncuri. We are
going to trust the user and assume that the user is in the branch
that he/she wants updated. We'll let the user manage branches with
git directly.
'''
git_cmd_opts = ""
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
writemsg_level(git_cmd + "\n")
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
previous_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**portage._native_kwargs(self.spawn_kwargs))
if exitcode != os.EX_OK:
msg = "!!! git pull error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
current_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
return (os.EX_OK, current_rev != previous_rev)
def _clean_exit(self, clean_phase):
if self._default_exit(clean_phase) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
dir_path = self._build_dir.dir_path
infloc = self._infloc
pkg = self.pkg
pkg_path = self._pkg_path
dir_mode = 0o755
for mydir in (dir_path, self._image_dir, infloc):
portage.util.ensure_dirs(mydir, uid=portage.data.portage_uid,
gid=portage.data.portage_gid, mode=dir_mode)
# This initializes PORTAGE_LOG_FILE.
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
self._writemsg_level(">>> Extracting info\n")
pkg_xpak = portage.xpak.tbz2(self._pkg_path)
check_missing_metadata = ("CATEGORY", "PF")
missing_metadata = set()
for k in check_missing_metadata:
v = pkg_xpak.getfile(_unicode_encode(k,
encoding=_encodings['repo.content']))
if not v:
missing_metadata.add(k)
pkg_xpak.unpackinfo(infloc)
for k in missing_metadata:
if k == "CATEGORY":
v = pkg.category
elif k == "PF":
v = pkg.pf
else:
continue
f = codecs.open(_unicode_encode(os.path.join(infloc, k),
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['content'], errors='replace')
try:
f.write(v + "\n")
finally:
f.close()
# Store the md5sum in the vdb.
f = codecs.open(_unicode_encode(os.path.join(infloc, 'BINPKGMD5'),
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['content'], errors='strict')
try:
f.write(str(portage.checksum.perform_md5(pkg_path)) + "\n")
finally:
f.close()
env_extractor = BinpkgEnvExtractor(background=self.background,
scheduler=self.scheduler, settings=self.settings)
self._start_task(env_extractor, self._env_extractor_exit)
def multiBuilder(self, options, settings, trees):
rValue = {}
directory = options.get("directory",
os.path.join(settings["PORTAGE_CONFIGROOT"],
USER_CONFIG_PATH, "sets"))
name_pattern = options.get("name_pattern", "${name}")
if not "$name" in name_pattern and not "${name}" in name_pattern:
raise SetConfigError(_("name_pattern doesn't include ${name} placeholder"))
greedy = get_boolean(options, "greedy", False)
# look for repository path variables
match = self._repopath_match.match(directory)
if match:
try:
directory = self._repopath_sub.sub(trees["porttree"].dbapi.treemap[match.groupdict()["reponame"]], directory)
except KeyError:
raise SetConfigError(_("Could not find repository '%s'") % match.groupdict()["reponame"])
try:
directory = _unicode_decode(directory,
encoding=_encodings['fs'], errors='strict')
# Now verify that we can also encode it.
_unicode_encode(directory,
encoding=_encodings['fs'], errors='strict')
except UnicodeError:
directory = _unicode_decode(directory,
encoding=_encodings['fs'], errors='replace')
raise SetConfigError(
_("Directory path contains invalid character(s) for encoding '%s': '%s'") \
% (_encodings['fs'], directory))
if os.path.isdir(directory):
directory = normalize_path(directory)
for parent, dirs, files in os.walk(directory):
try:
parent = _unicode_decode(parent,
encoding=_encodings['fs'], errors='strict')
except UnicodeDecodeError:
continue
for d in dirs[:]:
if d[:1] == '.':
dirs.remove(d)
for filename in files:
try:
filename = _unicode_decode(filename,
encoding=_encodings['fs'], errors='strict')
except UnicodeDecodeError:
continue
if filename[:1] == '.':
continue
if filename.endswith(".metadata"):
continue
filename = os.path.join(parent,
filename)[1 + len(directory):]
myname = name_pattern.replace("$name", filename)
myname = myname.replace("${name}", filename)
rValue[myname] = StaticFileSet(
os.path.join(directory, filename),
greedy=greedy, dbapi=trees["vartree"].dbapi)
return rValue
def unpackinfo(self, mydest): """Unpacks all the files from the dataSegment into 'mydest'.""" if not self.scan(): return 0 mydest = normalize_path(mydest) + os.sep a = open(_unicode_encode(self.file, encoding=_encodings['fs'], errors='strict'), 'rb') if not os.path.exists(mydest): os.makedirs(mydest) startpos = 0 while ((startpos + 8) < self.indexsize): namelen = decodeint(self.index[startpos:startpos + 4]) datapos = decodeint(self.index[startpos + 4 + namelen:startpos + 8 + namelen]) datalen = decodeint(self.index[startpos + 8 + namelen:startpos + 12 + namelen]) myname = self.index[startpos + 4:startpos + 4 + namelen] myname = _unicode_decode(myname, encoding=_encodings['repo.content'], errors='replace') filename = os.path.join(mydest, myname.lstrip(os.sep)) filename = normalize_path(filename) if not filename.startswith(mydest): # myname contains invalid ../ component(s) continue dirname = os.path.dirname(filename) if dirname: if not os.path.exists(dirname): os.makedirs(dirname) mydat = open(_unicode_encode(filename, encoding=_encodings['fs'], errors='strict'), 'wb') a.seek(self.datapos + datapos) mydat.write(a.read(datalen)) mydat.close() startpos = startpos + namelen + 12 a.close() return 1
def xsplit_mem(mydat):
if mydat[0:8] != _unicode_encode('XPAKPACK'):
return None
if mydat[-8:] != _unicode_encode('XPAKSTOP'):
return None
indexsize=decodeint(mydat[8:12])
return (mydat[16:indexsize+16], mydat[indexsize+16:-8])
def main():
TEST_FILE = _unicode_encode('__test__',
encoding=_encodings['fs'], errors='strict')
svn_dirname = _unicode_encode('.svn',
encoding=_encodings['fs'], errors='strict')
suite = unittest.TestSuite()
basedir = os.path.dirname(os.path.realpath(__file__))
testDirs = []
# the os.walk help mentions relative paths as being quirky
# I was tired of adding dirs to the list, so now we add __test__
# to each dir we want tested.
for root, dirs, files in os.walk(basedir):
if svn_dirname in dirs:
dirs.remove(svn_dirname)
try:
root = _unicode_decode(root,
encoding=_encodings['fs'], errors='strict')
except UnicodeDecodeError:
continue
if TEST_FILE in files:
testDirs.append(root)
for mydir in testDirs:
suite.addTests(getTests(os.path.join(basedir, mydir), basedir) )
return TextTestRunner(verbosity=2).run(suite)
def xpak(rootdir, outfile=None):
"""(rootdir, outfile) -- creates an xpak segment of the directory 'rootdir'
and under the name 'outfile' if it is specified. Otherwise it returns the
xpak segment."""
mylist = []
addtolist(mylist, rootdir)
mylist.sort()
mydata = {}
for x in mylist:
if x == 'CONTENTS':
# CONTENTS is generated during the merge process.
continue
x = _unicode_encode(x, encoding=_encodings['fs'], errors='strict')
with open(os.path.join(rootdir, x), 'rb') as f:
mydata[x] = f.read()
xpak_segment = xpak_mem(mydata)
if outfile:
outf = open(_unicode_encode(outfile,
encoding=_encodings['fs'], errors='strict'), 'wb')
outf.write(xpak_segment)
outf.close()
else:
return xpak_segment
def __init__(self, filename, mode='w', follow_links=True, **kargs):
"""Opens a temporary filename.pid in the same directory as filename."""
ObjectProxy.__init__(self)
object.__setattr__(self, '_aborted', False)
if 'b' in mode:
open_func = open
else:
open_func = codecs.open
kargs.setdefault('encoding', _encodings['content'])
kargs.setdefault('errors', 'backslashreplace')
if follow_links:
canonical_path = os.path.realpath(filename)
object.__setattr__(self, '_real_name', canonical_path)
tmp_name = "%s.%i" % (canonical_path, os.getpid())
try:
object.__setattr__(self, '_file',
open_func(_unicode_encode(tmp_name,
encoding=_encodings['fs'], errors='strict'),
mode=mode, **kargs))
return
except IOError as e:
if canonical_path == filename:
raise
writemsg(_("!!! Failed to open file: '%s'\n") % tmp_name,
noiselevel=-1)
writemsg("!!! %s\n" % str(e), noiselevel=-1)
object.__setattr__(self, '_real_name', filename)
tmp_name = "%s.%i" % (filename, os.getpid())
object.__setattr__(self, '_file',
open_func(_unicode_encode(tmp_name,
encoding=_encodings['fs'], errors='strict'),
mode=mode, **kargs))
def save_cache(logger, to_save={}, temp_path=DEFAULTS['DEFAULT_TMP_DIR']):
''' Tries to store caching information.
@param logger
@param to_save have to be dict with keys:
libraries, la_libraries, libraries_links and binaries
'''
if not os.path.exists(temp_path):
os.makedirs(temp_path)
try:
_file = open(_unicode_encode(os.path.join(temp_path, 'timestamp'),
encoding=_encodings['fs']), mode='w', encoding=_encodings['content'])
_file.write(_unicode(int(time.time())))
_file.close()
for key,val in to_save.items():
_file = open(_unicode_encode(os.path.join(temp_path, key),
encoding=_encodings['fs']), mode='w',
encoding=_encodings['content'])
for line in val:
_file.write(line + '\n')
_file.close()
except Exception as ex:
logger.warning('\t' + red('Could not save cache: %s' %str(ex)))
def aux_update(self, cpv, values): if not self.bintree.populated: self.bintree.populate() build_id = None try: build_id = cpv.build_id except AttributeError: if self.bintree._multi_instance: # The cpv.build_id attribute is required if we are in # multi-instance mode, since otherwise we won't know # which instance to update. raise else: cpv = self._instance_key(cpv, support_string=True)[0] build_id = cpv.build_id tbz2path = self.bintree.getname(cpv) if not os.path.exists(tbz2path): raise KeyError(cpv) mytbz2 = portage.xpak.tbz2(tbz2path) mydata = mytbz2.get_data() for k, v in values.items(): k = _unicode_encode(k, encoding=_encodings['repo.content'], errors='backslashreplace') v = _unicode_encode(v, encoding=_encodings['repo.content'], errors='backslashreplace') mydata[k] = v for k, v in list(mydata.items()): if not v: del mydata[k] mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) # inject will clear stale caches via cpv_inject. self.bintree.inject(cpv, filename=tbz2path)
def _setitem(self, cpv, values):
s = cpv.rfind("/")
fp=os.path.join(self.location,cpv[:s],".update.%i.%s" % (os.getpid(), cpv[s+1:]))
try:
myf = codecs.open(_unicode_encode(fp,
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['repo.content'],
errors='backslashreplace')
except (OSError, IOError) as e:
if errno.ENOENT == e.errno:
try:
self._ensure_dirs(cpv)
myf = codecs.open(_unicode_encode(fp,
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['repo.content'],
errors='backslashreplace')
except (OSError, IOError) as e:
raise cache_errors.CacheCorruption(cpv, e)
else:
raise cache_errors.CacheCorruption(cpv, e)
for x in self.auxdbkey_order:
myf.write(values.get(x,"")+"\n")
myf.close()
self._ensure_access(fp, mtime=values["_mtime_"])
#update written. now we move it.
new_fp = os.path.join(self.location,cpv)
try:
os.rename(fp, new_fp)
except (OSError, IOError) as e:
os.remove(fp)
raise cache_errors.CacheCorruption(cpv, e)
def xpak_mem(mydata):
"""Create an xpack segment from a map object."""
mydata_encoded = {}
for k, v in mydata.items():
k = _unicode_encode(k,
encoding=_encodings['repo.content'], errors='backslashreplace')
v = _unicode_encode(v,
encoding=_encodings['repo.content'], errors='backslashreplace')
mydata_encoded[k] = v
mydata = mydata_encoded
del mydata_encoded
indexglob = b''
indexpos = 0
dataglob = b''
datapos = 0
for x, newglob in mydata.items():
mydatasize = len(newglob)
indexglob = indexglob + encodeint(len(x)) + x + encodeint(datapos) + encodeint(mydatasize)
indexpos = indexpos + 4 + len(x) + 4 + 4
dataglob = dataglob + newglob
datapos = datapos + mydatasize
return b'XPAKPACK' \
+ encodeint(len(indexglob)) \
+ encodeint(len(dataglob)) \
+ indexglob \
+ dataglob \
+ b'XPAKSTOP'
def findname2(self, mycpv, mytree=None):
"""
Returns the location of the CPV, and what overlay it was in.
Searches overlays first, then PORTDIR; this allows us to return the first
matching file. As opposed to starting in portdir and then doing overlays
second, we would have to exhaustively search the overlays until we found
the file we wanted.
"""
if not mycpv:
return (None, 0)
mysplit = mycpv.split("/")
psplit = pkgsplit(mysplit[1])
if psplit is None or len(mysplit) != 2:
raise InvalidPackageName(mycpv)
# For optimal performace in this hot spot, we do manual unicode
# handling here instead of using the wrapped os module.
encoding = _encodings['fs']
errors = 'strict'
if mytree:
mytrees = [mytree]
else:
mytrees = self.porttrees[:]
mytrees.reverse()
relative_path = mysplit[0] + _os.sep + psplit[0] + _os.sep + \
mysplit[1] + ".ebuild"
if 'parse-eapi-glep-55' in self.doebuild_settings.features:
glep55_startswith = '%s.ebuild-' % mysplit[1]
for x in mytrees:
filename = x + _os.sep + relative_path
if _os.access(_unicode_encode(filename,
encoding=encoding, errors=errors), _os.R_OK):
return (filename, x)
pkgdir = _os.path.join(x, mysplit[0], psplit[0])
try:
files = _os.listdir(_unicode_encode(pkgdir,
encoding=encoding, errors=errors))
except OSError:
continue
for y in files:
try:
y = _unicode_decode(y, encoding=encoding, errors=errors)
except UnicodeDecodeError:
continue
if y.startswith(glep55_startswith):
return (_os.path.join(pkgdir, y), x)
else:
for x in mytrees:
filename = x + _os.sep + relative_path
if _os.access(_unicode_encode(filename,
encoding=encoding, errors=errors), _os.R_OK):
return (filename, x)
return (None, 0)
def update(self):
''' Update existing git repository, and ignore the syncuri. We are
going to trust the user and assume that the user is in the branch
that he/she wants updated. We'll let the user manage branches with
git directly.
'''
git_cmd_opts = ""
quiet = self.settings.get("PORTAGE_QUIET") == "1"
if quiet:
git_cmd_opts += " --quiet"
if self.repo.module_specific_options.get('sync-git-pull-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options['sync-git-pull-extra-opts']
if self.repo.sync_depth is None:
git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
else:
# Since the default merge strategy typically fails when
# the depth is not unlimited, use `git fetch` followed by
# `git reset --merge`.
remote_branch = portage._unicode_decode(
subprocess.check_output([self.bin_command, 'rev-parse',
'--abbrev-ref', '--symbolic-full-name', '@{upstream}'],
cwd=portage._unicode_encode(self.repo.location))).rstrip('\n')
git_cmd_opts += " --depth %d" % self.repo.sync_depth
git_cmd = "%s fetch %s%s" % (self.bin_command,
remote_branch.partition('/')[0], git_cmd_opts)
writemsg_level(git_cmd + "\n")
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
previous_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**self.spawn_kwargs)
if exitcode == os.EX_OK and self.repo.sync_depth is not None:
reset_cmd = [self.bin_command, 'reset', '--merge', remote_branch]
if quiet:
reset_cmd.append('--quiet')
exitcode = subprocess.call(reset_cmd,
cwd=portage._unicode_encode(self.repo.location))
if exitcode != os.EX_OK:
msg = "!!! git pull error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
current_rev = subprocess.check_output(rev_cmd,
cwd=portage._unicode_encode(self.repo.location))
return (os.EX_OK, current_rev != previous_rev)
def _ebuild_exit(self, ebuild_process):
if self.phase == "install":
out = portage.StringIO()
log_path = self.settings.get("PORTAGE_LOG_FILE")
log_file = None
if log_path is not None:
log_file = codecs.open(_unicode_encode(log_path,
encoding=_encodings['fs'], errors='strict'),
mode='a', encoding=_encodings['content'], errors='replace')
try:
_check_build_log(self.settings, out=out)
msg = _unicode_decode(out.getvalue(),
encoding=_encodings['content'], errors='replace')
if msg:
if not self.background:
writemsg_stdout(msg, noiselevel=-1)
if log_file is not None:
log_file.write(msg)
finally:
if log_file is not None:
log_file.close()
if self._default_exit(ebuild_process) != os.EX_OK:
self._die_hooks()
return
settings = self.settings
if self.phase == "install":
out = None
log_path = self.settings.get("PORTAGE_LOG_FILE")
log_file = None
if self.background and log_path is not None:
log_file = codecs.open(_unicode_encode(log_path,
encoding=_encodings['fs'], errors='strict'),
mode='a', encoding=_encodings['content'], errors='replace')
out = log_file
_post_src_install_chost_fix(settings)
_post_src_install_uid_fix(settings, out=out)
if log_file is not None:
log_file.close()
post_phase_cmds = _post_phase_cmds.get(self.phase)
if post_phase_cmds is not None:
post_phase = MiscFunctionsProcess(background=self.background,
commands=post_phase_cmds, phase=self.phase, pkg=self.pkg,
scheduler=self.scheduler, settings=settings)
self._start_task(post_phase, self._post_phase_exit)
return
self.returncode = ebuild_process.returncode
self._current_task = None
self.wait()
def get_commit_message_with_editor(editor, message=None):
"""
Execute editor with a temporary file as it's argument
and return the file content afterwards.
@param editor: An EDITOR value from the environment
@type: string
@param message: An iterable of lines to show in the editor.
@type: iterable
@rtype: string or None
@return: A string on success or None if an error occurs.
"""
fd, filename = mkstemp()
try:
os.write(
fd,
_unicode_encode(
_(
"\n# Please enter the commit message "
+ "for your changes.\n# (Comment lines starting "
+ "with '#' will not be included)\n"
),
encoding=_encodings["content"],
errors="backslashreplace",
),
)
if message:
os.write(fd, b"#\n")
for line in message:
os.write(fd, _unicode_encode("#" + line, encoding=_encodings["content"], errors="backslashreplace"))
os.close(fd)
retval = os.system(editor + " '%s'" % filename)
if not (os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == os.EX_OK):
return None
try:
with io.open(
_unicode_encode(filename, encoding=_encodings["fs"], errors="strict"),
mode="r",
encoding=_encodings["content"],
errors="replace",
) as f:
mylines = f.readlines()
except OSError as e:
if e.errno != errno.ENOENT:
raise
del e
return None
return "".join(line for line in mylines if not line.startswith("#"))
finally:
try:
os.unlink(filename)
except OSError:
pass
def getindex(infile):
"""(infile) -- grabs the index segment from the infile and returns it."""
myfile = open(_unicode_encode(infile,
encoding=_encodings['fs'], errors='strict'), 'rb')
myheader=myfile.read(16)
if myheader[0:8] != _unicode_encode('XPAKPACK'):
myfile.close()
return
indexsize=decodeint(myheader[8:12])
myindex=myfile.read(indexsize)
myfile.close()
return myindex
def recompose_mem(self, xpdata):
self.scan() # Don't care about condition... We'll rewrite the data anyway.
myfile = open(_unicode_encode(self.file,
encoding=_encodings['fs'], errors='strict'), 'ab+')
if not myfile:
raise IOError
myfile.seek(-self.xpaksize,2) # 0,2 or -0,2 just mean EOF.
myfile.truncate()
myfile.write(xpdata+encodeint(len(xpdata)) + _unicode_encode('STOP'))
myfile.flush()
myfile.close()
return 1
def rename(src, dest):
src = _unicode_encode(src, encoding=_encodings['fs'], errors='strict')
dest = _unicode_encode(dest, encoding=_encodings['fs'], errors='strict')
(rc, ctx) = selinux.lgetfilecon(src)
if rc < 0:
src = _unicode_decode(src, encoding=_encodings['fs'], errors='replace')
raise OSError(_("rename: Failed getting context of \"%s\".") % src)
setfscreate(ctx)
try:
os.rename(src,dest)
finally:
setfscreate()
def get_commit_message_with_editor(editor, message=None, prefix=""):
"""
Execute editor with a temporary file as it's argument
and return the file content afterwards.
@param editor: An EDITOR value from the environment
@type: string
@param message: An iterable of lines to show in the editor.
@type: iterable
@param prefix: Suggested prefix for the commit message summary line.
@type: string
@rtype: string or None
@return: A string on success or None if an error occurs.
"""
commitmessagedir = tempfile.mkdtemp(".repoman.msg")
filename = os.path.join(commitmessagedir, "COMMIT_EDITMSG")
try:
with open(filename, "wb") as mymsg:
mymsg.write(
_unicode_encode(_(
prefix +
"\n\n# Please enter the commit message "
"for your changes.\n# (Comment lines starting "
"with '#' will not be included)\n"),
encoding=_encodings['content'], errors='backslashreplace'))
if message:
mymsg.write(b"#\n")
for line in message:
mymsg.write(
_unicode_encode(
"#" + line, encoding=_encodings['content'],
errors='backslashreplace'))
retval = os.system(editor + " '%s'" % filename)
if not (os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == os.EX_OK):
return None
try:
with io.open(_unicode_encode(
filename, encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['content'], errors='replace') as f:
mylines = f.readlines()
except OSError as e:
if e.errno != errno.ENOENT:
raise
del e
return None
return "".join(line for line in mylines if not line.startswith("#"))
finally:
try:
shutil.rmtree(commitmessagedir)
except OSError:
pass
def create_message(sender, recipient, subject, body, attachments=None):
from email.header import Header
from email.mime.base import MIMEBase as BaseMessage
from email.mime.multipart import MIMEMultipart as MultipartMessage
from email.utils import formatdate
if sys.hexversion < 0x3000000:
sender = _unicode_encode(sender,
encoding=_encodings['content'], errors='strict')
recipient = _unicode_encode(recipient,
encoding=_encodings['content'], errors='strict')
subject = _unicode_encode(subject,
encoding=_encodings['content'], errors='backslashreplace')
body = _unicode_encode(body,
encoding=_encodings['content'], errors='backslashreplace')
if attachments == None:
mymessage = TextMessage(body)
else:
mymessage = MultipartMessage()
mymessage.attach(TextMessage(body))
for x in attachments:
if isinstance(x, BaseMessage):
mymessage.attach(x)
elif isinstance(x, basestring):
if sys.hexversion < 0x3000000:
x = _unicode_encode(x,
encoding=_encodings['content'],
errors='backslashreplace')
mymessage.attach(TextMessage(x))
else:
raise portage.exception.PortageException(_("Can't handle type of attachment: %s") % type(x))
mymessage.set_unixfrom(sender)
mymessage["To"] = recipient
mymessage["From"] = sender
# Use Header as a workaround so that long subject lines are wrapped
# correctly by <=python-2.6 (gentoo bug #263370, python issue #1974).
# Also, need to force ascii for python3, in order to avoid
# UnicodeEncodeError with non-ascii characters:
# File "/usr/lib/python3.1/email/header.py", line 189, in __init__
# self.append(s, charset, errors)
# File "/usr/lib/python3.1/email/header.py", line 262, in append
# input_bytes = s.encode(input_charset, errors)
#UnicodeEncodeError: 'ascii' codec can't encode characters in position 0-9: ordinal not in range(128)
mymessage["Subject"] = Header(_force_ascii_if_necessary(subject))
mymessage["Date"] = formatdate(localtime=True)
return mymessage
def testCompileModules(self):
iters = [os.walk(os.path.join(PORTAGE_PYM_PATH, x))
for x in PORTAGE_PYM_PACKAGES]
iters.append(os.walk(PORTAGE_BIN_PATH))
for parent, _dirs, files in itertools.chain(*iters):
parent = _unicode_decode(parent,
encoding=_encodings['fs'], errors='strict')
for x in files:
x = _unicode_decode(x,
encoding=_encodings['fs'], errors='strict')
if x[-4:] in ('.pyc', '.pyo'):
continue
x = os.path.join(parent, x)
st = os.lstat(x)
if not stat.S_ISREG(st.st_mode):
continue
bin_path = os.path.relpath(x, PORTAGE_BIN_PATH)
mod_path = os.path.relpath(x, PORTAGE_PYM_PATH)
meta = module_metadata.get(mod_path) or script_metadata.get(bin_path)
if meta:
req_py = tuple(int(x) for x
in meta.get('required_python', '0.0').split('.'))
if sys.version_info < req_py:
continue
do_compile = False
if x[-3:] == '.py':
do_compile = True
else:
# Check for python shebang.
try:
with open(_unicode_encode(x,
encoding=_encodings['fs'], errors='strict'), 'rb') as f:
line = _unicode_decode(f.readline(),
encoding=_encodings['content'], errors='replace')
except IOError as e:
# Some tests create files that are unreadable by the
# user (by design), so ignore EACCES issues.
if e.errno != errno.EACCES:
raise
continue
if line[:2] == '#!' and 'python' in line:
do_compile = True
if do_compile:
with open(_unicode_encode(x,
encoding=_encodings['fs'], errors='strict'), 'rb') as f:
compile(f.read(), x, 'exec')
def get_global_useflags():
"""Get global and expanded USE flag variables from
PORTDIR/profiles/use.desc and PORTDIR/profiles/desc/*.desc respectively.
@rtype: dict
@return: {'flag_name': 'flag description', ...}
"""
global_usedesc = {}
# Get global USE flag descriptions
try:
path = os.path.join(settings["PORTDIR"], 'profiles', 'use.desc')
with open(_unicode_encode(path, encoding=_encodings['fs']),
encoding=_encodings['content']) as open_file:
for line in open_file:
if line.startswith('#'):
continue
# Ex. of fields: ['syslog', 'Enables support for syslog\n']
fields = line.split(" - ", 1)
if len(fields) == 2:
global_usedesc[fields[0]] = fields[1].rstrip()
except IOError:
sys.stderr.write(
pp.warn(
"Could not load USE flag descriptions from %s" % pp.path(path)
)
)
del path, open_file
# Add USE_EXPANDED variables to usedesc hash -- Bug #238005
for path in glob(os.path.join(settings["PORTDIR"],
'profiles', 'desc', '*.desc')):
try:
with open(_unicode_encode(path, encoding=_encodings['fs']),
encoding=_encodings['content']) as open_file:
for line in open_file:
if line.startswith('#'):
continue
fields = [field.strip() for field in line.split(" - ", 1)]
if len(fields) == 2:
expanded_useflag = "%s_%s" % \
(path.split("/")[-1][0:-5], fields[0])
global_usedesc[expanded_useflag] = fields[1]
except IOError:
sys.stderr.write(
pp.warn("Could not load USE flag descriptions from %s" % path)
)
return global_usedesc
def getboth(infile):
"""(infile) -- grabs the index and data segments from the infile.
Returns an array [indexSegment,dataSegment]"""
myfile = open(_unicode_encode(infile,
encoding=_encodings['fs'], errors='strict'), 'rb')
myheader=myfile.read(16)
if myheader[0:8] != _unicode_encode('XPAKPACK'):
myfile.close()
return
indexsize=decodeint(myheader[8:12])
datasize=decodeint(myheader[12:16])
myindex=myfile.read(indexsize)
mydata=myfile.read(datasize)
myfile.close()
return myindex, mydata
def store(self):
"""
Store the registry data to the file. The existing inode will be
replaced atomically, so if that inode is currently being used
for a lock then that lock will be rendered useless. Therefore,
it is important not to call this method until the current lock
is ready to be immediately released.
"""
if os.environ.get("SANDBOX_ON") == "1" or \
self._data == self._data_orig:
return
try:
f = atomic_ofstream(self._filename, 'wb')
if self._json_write:
f.write(_unicode_encode(
json.dumps(self._data, **self._json_write_opts),
encoding=_encodings['repo.content'], errors='strict'))
else:
pickle.dump(self._data, f, protocol=2)
f.close()
except EnvironmentError as e:
if e.errno != PermissionDenied.errno:
writemsg_level("!!! %s %s\n" % (e, self._filename),
level=logging.ERROR, noiselevel=-1)
else:
self._data_orig = self._data.copy()
def emergelog(xterm_titles, mystr, short_msg=None):
if _disable:
return
mystr = _unicode_decode(mystr)
if short_msg is not None:
short_msg = _unicode_decode(short_msg)
if xterm_titles and short_msg:
if "HOSTNAME" in os.environ:
short_msg = os.environ["HOSTNAME"]+": "+short_msg
xtermTitle(short_msg)
try:
file_path = os.path.join(_emerge_log_dir, 'emerge.log')
existing_log = os.path.exists(file_path)
mylogfile = io.open(_unicode_encode(file_path,
encoding=_encodings['fs'], errors='strict'),
mode='a', encoding=_encodings['content'],
errors='backslashreplace')
if not existing_log:
portage.util.apply_secpass_permissions(file_path,
uid=portage.portage_uid, gid=portage.portage_gid,
mode=0o660)
mylock = portage.locks.lockfile(file_path)
try:
mylogfile.write("%.0f: %s\n" % (time.time(), mystr))
mylogfile.close()
finally:
portage.locks.unlockfile(mylock)
except (IOError,OSError,portage.exception.PortageException) as e:
if secpass >= 1:
portage.util.writemsg("emergelog(): %s\n" % (e,), noiselevel=-1)
def mkdir(target, refdir):
target = _unicode_encode(target, encoding=_encodings['fs'], errors='strict')
refdir = _unicode_encode(refdir, encoding=_encodings['fs'], errors='strict')
(rc, ctx) = selinux.getfilecon(refdir)
if rc < 0:
refdir = _unicode_decode(refdir, encoding=_encodings['fs'],
errors='replace')
raise OSError(
_("mkdir: Failed getting context of reference directory \"%s\".") \
% refdir)
setfscreate(ctx)
try:
os.mkdir(target)
finally:
setfscreate()
def _adjust_perms_msg(settings, msg):
def write(msg):
writemsg(msg, noiselevel=-1)
background = settings.get("PORTAGE_BACKGROUND") == "1"
log_path = settings.get("PORTAGE_LOG_FILE")
log_file = None
log_file_real = None
if background and log_path is not None:
try:
log_file = open(_unicode_encode(log_path,
encoding=_encodings['fs'], errors='strict'), mode='ab')
log_file_real = log_file
except IOError:
def write(msg):
pass
else:
if log_path.endswith('.gz'):
log_file = gzip.GzipFile(filename='',
mode='ab', fileobj=log_file)
def write(msg):
log_file.write(_unicode_encode(msg))
log_file.flush()
try:
write(msg)
finally:
if log_file is not None:
log_file.close()
if log_file_real is not log_file:
log_file_real.close()
def _env_update(makelinks, target_root, prev_mtimes, contents, env,
writemsg_level):
if writemsg_level is None:
writemsg_level = portage.util.writemsg_level
if target_root is None:
target_root = portage.settings["ROOT"]
if prev_mtimes is None:
prev_mtimes = portage.mtimedb["ldpath"]
if env is None:
settings = portage.settings
else:
settings = env
eprefix = settings.get("EPREFIX", "")
eprefix_lstrip = eprefix.lstrip(os.sep)
eroot = normalize_path(os.path.join(target_root, eprefix_lstrip)).rstrip(os.sep) + os.sep
envd_dir = os.path.join(eroot, "etc", "env.d")
ensure_dirs(envd_dir, mode=0o755)
fns = listdir(envd_dir, EmptyOnError=1)
fns.sort()
templist = []
for x in fns:
if len(x) < 3:
continue
if not x[0].isdigit() or not x[1].isdigit():
continue
if x.startswith(".") or x.endswith("~") or x.endswith(".bak"):
continue
templist.append(x)
fns = templist
del templist
space_separated = set(["CONFIG_PROTECT", "CONFIG_PROTECT_MASK"])
colon_separated = set(["ADA_INCLUDE_PATH", "ADA_OBJECTS_PATH",
"CLASSPATH", "INFODIR", "INFOPATH", "KDEDIRS", "LDPATH", "MANPATH",
"PATH", "PKG_CONFIG_PATH", "PRELINK_PATH", "PRELINK_PATH_MASK",
"PYTHONPATH", "ROOTPATH"])
config_list = []
for x in fns:
file_path = os.path.join(envd_dir, x)
try:
myconfig = getconfig(file_path, expand=False)
except ParseError as e:
writemsg("!!! '%s'\n" % str(e), noiselevel=-1)
del e
continue
if myconfig is None:
# broken symlink or file removed by a concurrent process
writemsg("!!! File Not Found: '%s'\n" % file_path, noiselevel=-1)
continue
config_list.append(myconfig)
if "SPACE_SEPARATED" in myconfig:
space_separated.update(myconfig["SPACE_SEPARATED"].split())
del myconfig["SPACE_SEPARATED"]
if "COLON_SEPARATED" in myconfig:
colon_separated.update(myconfig["COLON_SEPARATED"].split())
del myconfig["COLON_SEPARATED"]
env = {}
specials = {}
for var in space_separated:
mylist = []
for myconfig in config_list:
if var in myconfig:
for item in myconfig[var].split():
if item and not item in mylist:
mylist.append(item)
del myconfig[var] # prepare for env.update(myconfig)
if mylist:
env[var] = " ".join(mylist)
specials[var] = mylist
for var in colon_separated:
mylist = []
for myconfig in config_list:
if var in myconfig:
for item in myconfig[var].split(":"):
if item and not item in mylist:
mylist.append(item)
del myconfig[var] # prepare for env.update(myconfig)
if mylist:
env[var] = ":".join(mylist)
specials[var] = mylist
for myconfig in config_list:
"""Cumulative variables have already been deleted from myconfig so that
they won't be overwritten by this dict.update call."""
env.update(myconfig)
ldsoconf_path = os.path.join(eroot, "etc", "ld.so.conf")
try:
myld = io.open(_unicode_encode(ldsoconf_path,
encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['content'], errors='replace')
myldlines = myld.readlines()
myld.close()
oldld = []
for x in myldlines:
#each line has at least one char (a newline)
if x[:1] == "#":
continue
oldld.append(x[:-1])
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
oldld = None
newld = specials["LDPATH"]
if (oldld != newld):
#ld.so.conf needs updating and ldconfig needs to be run
myfd = atomic_ofstream(ldsoconf_path)
myfd.write("# ld.so.conf autogenerated by env-update; make all changes to\n")
myfd.write("# contents of /etc/env.d directory\n")
for x in specials["LDPATH"]:
myfd.write(x + "\n")
myfd.close()
potential_lib_dirs = set()
for lib_dir_glob in ('usr/lib*', 'lib*'):
x = os.path.join(eroot, lib_dir_glob)
for y in glob.glob(_unicode_encode(x,
encoding=_encodings['fs'], errors='strict')):
try:
y = _unicode_decode(y,
encoding=_encodings['fs'], errors='strict')
except UnicodeDecodeError:
continue
if os.path.basename(y) != 'libexec':
potential_lib_dirs.add(y[len(eroot):])
# Update prelink.conf if we are prelink-enabled
if prelink_capable:
prelink_d = os.path.join(eroot, 'etc', 'prelink.conf.d')
ensure_dirs(prelink_d)
newprelink = atomic_ofstream(os.path.join(prelink_d, 'portage.conf'))
newprelink.write("# prelink.conf autogenerated by env-update; make all changes to\n")
newprelink.write("# contents of /etc/env.d directory\n")
for x in sorted(potential_lib_dirs) + ['bin', 'sbin']:
newprelink.write('-l /%s\n' % (x,));
prelink_paths = set()
prelink_paths |= set(specials.get('LDPATH', []))
prelink_paths |= set(specials.get('PATH', []))
prelink_paths |= set(specials.get('PRELINK_PATH', []))
prelink_path_mask = specials.get('PRELINK_PATH_MASK', [])
for x in prelink_paths:
if not x:
continue
if x[-1:] != '/':
x += "/"
plmasked = 0
for y in prelink_path_mask:
if not y:
continue
if y[-1] != '/':
y += "/"
if y == x[0:len(y)]:
plmasked = 1
break
if not plmasked:
newprelink.write("-h %s\n" % (x,))
for x in prelink_path_mask:
newprelink.write("-b %s\n" % (x,))
newprelink.close()
# Migration code path. If /etc/prelink.conf was generated by us, then
# point it to the new stuff until the prelink package re-installs.
prelink_conf = os.path.join(eroot, 'etc', 'prelink.conf')
try:
with open(_unicode_encode(prelink_conf,
encoding=_encodings['fs'], errors='strict'), 'rb') as f:
if f.readline() == b'# prelink.conf autogenerated by env-update; make all changes to\n':
f = atomic_ofstream(prelink_conf)
f.write('-c /etc/prelink.conf.d/*.conf\n')
f.close()
except IOError as e:
if e.errno != errno.ENOENT:
raise
current_time = long(time.time())
mtime_changed = False
lib_dirs = set()
for lib_dir in set(specials['LDPATH']) | potential_lib_dirs:
x = os.path.join(eroot, lib_dir.lstrip(os.sep))
try:
newldpathtime = os.stat(x)[stat.ST_MTIME]
lib_dirs.add(normalize_path(x))
except OSError as oe:
if oe.errno == errno.ENOENT:
try:
del prev_mtimes[x]
except KeyError:
pass
# ignore this path because it doesn't exist
continue
raise
if newldpathtime == current_time:
# Reset mtime to avoid the potential ambiguity of times that
# differ by less than 1 second.
newldpathtime -= 1
os.utime(x, (newldpathtime, newldpathtime))
prev_mtimes[x] = newldpathtime
mtime_changed = True
elif x in prev_mtimes:
if prev_mtimes[x] == newldpathtime:
pass
else:
prev_mtimes[x] = newldpathtime
mtime_changed = True
else:
prev_mtimes[x] = newldpathtime
mtime_changed = True
if makelinks and \
not mtime_changed and \
contents is not None:
libdir_contents_changed = False
for mypath, mydata in contents.items():
if mydata[0] not in ("obj", "sym"):
continue
head, tail = os.path.split(mypath)
if head in lib_dirs:
libdir_contents_changed = True
break
if not libdir_contents_changed:
makelinks = False
ldconfig = "/sbin/ldconfig"
if "CHOST" in settings and "CBUILD" in settings and \
settings["CHOST"] != settings["CBUILD"]:
ldconfig = find_binary("%s-ldconfig" % settings["CHOST"])
# Only run ldconfig as needed
if makelinks and ldconfig and not eprefix:
# ldconfig has very different behaviour between FreeBSD and Linux
if ostype == "Linux" or ostype.lower().endswith("gnu"):
# We can't update links if we haven't cleaned other versions first, as
# an older package installed ON TOP of a newer version will cause ldconfig
# to overwrite the symlinks we just made. -X means no links. After 'clean'
# we can safely create links.
writemsg_level(_(">>> Regenerating %setc/ld.so.cache...\n") % \
(target_root,))
os.system("cd / ; %s -X -r '%s'" % (ldconfig, target_root))
elif ostype in ("FreeBSD", "DragonFly"):
writemsg_level(_(">>> Regenerating %svar/run/ld-elf.so.hints...\n") % \
target_root)
os.system(("cd / ; %s -elf -i " + \
"-f '%svar/run/ld-elf.so.hints' '%setc/ld.so.conf'") % \
(ldconfig, target_root, target_root))
del specials["LDPATH"]
penvnotice = "# THIS FILE IS AUTOMATICALLY GENERATED BY env-update.\n"
penvnotice += "# DO NOT EDIT THIS FILE. CHANGES TO STARTUP PROFILES\n"
cenvnotice = penvnotice[:]
penvnotice += "# GO INTO /etc/profile NOT /etc/profile.env\n\n"
cenvnotice += "# GO INTO /etc/csh.cshrc NOT /etc/csh.env\n\n"
#create /etc/profile.env for bash support
outfile = atomic_ofstream(os.path.join(eroot, "etc", "profile.env"))
outfile.write(penvnotice)
env_keys = [x for x in env if x != "LDPATH"]
env_keys.sort()
for k in env_keys:
v = env[k]
if v.startswith('$') and not v.startswith('${'):
outfile.write("export %s=$'%s'\n" % (k, v[1:]))
else:
outfile.write("export %s='%s'\n" % (k, v))
outfile.close()
#create /etc/csh.env for (t)csh support
outfile = atomic_ofstream(os.path.join(eroot, "etc", "csh.env"))
outfile.write(cenvnotice)
for x in env_keys:
outfile.write("setenv %s '%s'\n" % (x, env[x]))
outfile.close()
def perform_checksum(filename, hashname="MD5", calc_prelink=0):
"""
Run a specific checksum against a file. The filename can
be either unicode or an encoded byte string. If filename
is unicode then a UnicodeDecodeError will be raised if
necessary.
@param filename: File to run the checksum against
@type filename: String
@param hashname: The type of hash function to run
@type hashname: String
@param calc_prelink: Whether or not to reverse prelink before running the checksum
@type calc_prelink: Integer
@rtype: Tuple
@return: The hash and size of the data
"""
global prelink_capable
# Make sure filename is encoded with the correct encoding before
# it is passed to spawn (for prelink) and/or the hash function.
filename = _unicode_encode(filename,
encoding=_encodings['fs'],
errors='strict')
myfilename = filename
prelink_tmpfile = None
try:
if (calc_prelink and prelink_capable and is_prelinkable_elf(filename)):
# Create non-prelinked temporary file to checksum.
# Files rejected by prelink are summed in place.
try:
tmpfile_fd, prelink_tmpfile = tempfile.mkstemp()
try:
retval = portage.process.spawn(
[PRELINK_BINARY, "--verify", filename],
fd_pipes={1: tmpfile_fd})
finally:
os.close(tmpfile_fd)
if retval == os.EX_OK:
myfilename = prelink_tmpfile
except portage.exception.CommandNotFound:
# This happens during uninstallation of prelink.
prelink_capable = False
try:
if hashname not in hashfunc_keys:
raise portage.exception.DigestException(hashname + \
" hash function not available (needs dev-python/pycrypto)")
myhash, mysize = hashfunc_map[hashname].checksum_file(myfilename)
except (OSError, IOError) as e:
if e.errno in (errno.ENOENT, errno.ESTALE):
raise portage.exception.FileNotFound(myfilename)
elif e.errno == portage.exception.PermissionDenied.errno:
raise portage.exception.PermissionDenied(myfilename)
raise
return myhash, mysize
finally:
if prelink_tmpfile:
try:
os.unlink(prelink_tmpfile)
except OSError as e:
if e.errno != errno.ENOENT:
raise
del e
def _fetch_uri(self, uri):
if self.config.options.dry_run:
# Simply report success.
logging.info("dry-run: fetch '%s' from '%s'" %
(self.distfile, uri))
self._success()
self.returncode = os.EX_OK
self._async_wait()
return
if self.config.options.temp_dir:
self._fetch_tmp_dir_info = 'temp-dir'
distdir = self.config.options.temp_dir
else:
self._fetch_tmp_dir_info = 'distfiles'
distdir = self.config.options.distfiles
tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % os.getpid()
variables = {"DISTDIR": distdir, "URI": uri, "FILE": tmp_basename}
self._fetch_tmp_file = os.path.join(distdir, tmp_basename)
try:
os.unlink(self._fetch_tmp_file)
except OSError:
pass
args = portage.util.shlex_split(default_fetchcommand)
args = [portage.util.varexpand(x, mydict=variables) for x in args]
if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
not os.path.isabs(args[0]):
# Python 3.1 _execvp throws TypeError for non-absolute executable
# path passed as bytes (see https://bugs.python.org/issue8513).
fullname = portage.process.find_binary(args[0])
if fullname is None:
raise portage.exception.CommandNotFound(args[0])
args[0] = fullname
args = [
_unicode_encode(x, encoding=_encodings['fs'], errors='strict')
for x in args
]
null_fd = os.open(os.devnull, os.O_RDONLY)
fetcher = PopenProcess(background=self.background,
proc=subprocess.Popen(args,
stdin=null_fd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT),
scheduler=self.scheduler)
os.close(null_fd)
fetcher.pipe_reader = PipeLogger(background=self.background,
input_fd=fetcher.proc.stdout,
log_file_path=self._log_path,
scheduler=self.scheduler)
self._start_task(fetcher, self._fetcher_exit)
def ExtractKernelVersion(base_dir):
"""
Try to figure out what kernel version we are running
@param base_dir: Path to sources (usually /usr/src/linux)
@type base_dir: string
@rtype: tuple( version[string], error[string])
@returns:
1. tuple( version[string], error[string])
Either version or error is populated (but never both)
"""
lines = []
pathname = os.path.join(base_dir, 'Makefile')
try:
f = codecs.open(_unicode_encode(pathname,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['content'],
errors='replace')
except OSError as details:
return (None, str(details))
except IOError as details:
return (None, str(details))
try:
for i in range(4):
lines.append(f.readline())
except OSError as details:
return (None, str(details))
except IOError as details:
return (None, str(details))
lines = [l.strip() for l in lines]
version = ''
#XXX: The following code relies on the ordering of vars within the Makefile
for line in lines:
# split on the '=' then remove annoying whitespace
items = line.split("=")
items = [i.strip() for i in items]
if items[0] == 'VERSION' or \
items[0] == 'PATCHLEVEL':
version += items[1]
version += "."
elif items[0] == 'SUBLEVEL':
version += items[1]
elif items[0] == 'EXTRAVERSION' and \
items[-1] != items[0]:
version += items[1]
# Grab a list of files named localversion* and sort them
localversions = os.listdir(base_dir)
for x in range(len(localversions) - 1, -1, -1):
if localversions[x][:12] != "localversion":
del localversions[x]
localversions.sort()
# Append the contents of each to the version string, stripping ALL whitespace
for lv in localversions:
version += "".join(" ".join(grabfile(base_dir + "/" + lv)).split())
# Check the .config for a CONFIG_LOCALVERSION and append that too, also stripping whitespace
kernelconfig = getconfig(base_dir + "/.config")
if kernelconfig and "CONFIG_LOCALVERSION" in kernelconfig:
version += "".join(kernelconfig["CONFIG_LOCALVERSION"].split())
return (version, None)
def detect_conflicts(options):
"""Determine if the checkout has cvs conflicts.
TODO(antarus): Also this should probably not call sys.exit() as
repoman is run on >1 packages and one failure should not cause
subsequent packages to fail.
Returns:
None (calls sys.exit on fatal problems)
"""
cmd = (r"cvs -n up 2>/dev/null | "
r"egrep '^[^\?] .*' | "
r"egrep -v '^. .*/digest-[^/]+|^cvs server: .* -- ignored$'")
msg = (
"Performing a %s with a little magic grep to check for updates." %
green("cvs -n up"))
logging.info(msg)
# Use Popen instead of getstatusoutput(), in order to avoid
# unicode handling problems (see bug #310789).
args = [BASH_BINARY, "-c", cmd]
args = [_unicode_encode(x) for x in args]
proc = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out = _unicode_decode(proc.communicate()[0])
proc.wait()
mylines = out.splitlines()
myupdates = []
for line in mylines:
if not line:
continue
# [ ] Unmodified (SVN) [U] Updates [P] Patches
# [M] Modified [A] Added [R] Removed / Replaced
# [D] Deleted
if line[0] not in " UPMARD":
# Stray Manifest is fine, we will readd it anyway.
if line[0] == '?' and line[1:].lstrip() == 'Manifest':
continue
logging.error(
red("!!! Please fix the following issues reported "
"from cvs: %s" % green("(U,P,M,A,R,D are ok)")))
logging.error(
red("!!! Note: This is a pretend/no-modify pass..."))
logging.error(out)
sys.exit(1)
elif line[0] in "UP":
myupdates.append(line[2:])
if myupdates:
logging.info(green("Fetching trivial updates..."))
if options.pretend:
logging.info("(cvs update " + " ".join(myupdates) + ")")
retval = os.EX_OK
else:
retval = os.system("cvs update " + " ".join(myupdates))
if retval != os.EX_OK:
logging.fatal("!!! cvs exited with an error. Terminating.")
sys.exit(retval)
return False
def collect_ebuild_messages(path):
""" Collect elog messages generated by the bash logging function stored
at 'path'.
"""
mylogfiles = None
try:
mylogfiles = os.listdir(path)
except OSError:
pass
# shortcut for packages without any messages
if not mylogfiles:
return {}
# exploit listdir() file order so we process log entries in chronological order
mylogfiles.reverse()
logentries = {}
for msgfunction in mylogfiles:
filename = os.path.join(path, msgfunction)
if msgfunction not in EBUILD_PHASES:
writemsg(_("!!! can't process invalid log file: %s\n") % filename,
noiselevel=-1)
continue
if not msgfunction in logentries:
logentries[msgfunction] = []
lastmsgtype = None
msgcontent = []
f = io.open(_unicode_encode(filename,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['repo.content'],
errors='replace')
# Use split('\n') since normal line iteration or readlines() will
# split on \r characters as shown in bug #390833.
for l in f.read().split('\n'):
if not l:
continue
try:
msgtype, msg = l.split(" ", 1)
if msgtype not in _log_levels:
raise ValueError(msgtype)
except ValueError:
writemsg(_("!!! malformed entry in "
"log file: '%s': %s\n") % (filename, l),
noiselevel=-1)
continue
if lastmsgtype is None:
lastmsgtype = msgtype
if msgtype == lastmsgtype:
msgcontent.append(msg)
else:
if msgcontent:
logentries[msgfunction].append((lastmsgtype, msgcontent))
msgcontent = [msg]
lastmsgtype = msgtype
f.close()
if msgcontent:
logentries[msgfunction].append((lastmsgtype, msgcontent))
# clean logfiles to avoid repetitions
for f in mylogfiles:
try:
os.unlink(os.path.join(path, f))
except OSError:
pass
return logentries
def _addProfile(self, currentPath, repositories, known_repos):
current_abs_path = os.path.abspath(currentPath)
allow_directories = True
allow_parent_colon = True
repo_loc = None
compat_mode = False
eapi_file = os.path.join(currentPath, "eapi")
eapi = "0"
f = None
try:
f = io.open(_unicode_encode(eapi_file,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['content'],
errors='replace')
eapi = f.readline().strip()
except IOError:
pass
else:
if not eapi_is_supported(eapi):
raise ParseError(_(
"Profile contains unsupported "
"EAPI '%s': '%s'") % \
(eapi, os.path.realpath(eapi_file),))
finally:
if f is not None:
f.close()
intersecting_repos = [
x for x in known_repos if current_abs_path.startswith(x[0])
]
if intersecting_repos:
# protect against nested repositories. Insane configuration, but the longest
# path will be the correct one.
repo_loc, layout_data = max(intersecting_repos,
key=lambda x: len(x[0]))
allow_directories = eapi_allows_directories_on_profile_level_and_repository_level(eapi) or \
any(x in _portage1_profiles_allow_directories for x in layout_data['profile-formats'])
compat_mode = not eapi_allows_directories_on_profile_level_and_repository_level(eapi) and \
layout_data['profile-formats'] == ('portage-1-compat',)
allow_parent_colon = any(x in _allow_parent_colon
for x in layout_data['profile-formats'])
if compat_mode:
offenders = _PORTAGE1_DIRECTORIES.intersection(
os.listdir(currentPath))
offenders = sorted(x for x in offenders
if os.path.isdir(os.path.join(currentPath, x)))
if offenders:
warnings.warn(
_("\nThe selected profile is implicitly using the 'portage-1' format:\n"
"\tprofile = %(profile_path)s\n"
"But this repository is not using that format:\n"
"\trepo = %(repo_name)s\n"
"This will break in the future. Please convert these dirs to files:\n"
"\t%(files)s\n"
"Or, add this line to the repository's layout.conf:\n"
"\tprofile-formats = portage-1") %
dict(profile_path=currentPath,
repo_name=repo_loc,
files='\n\t'.join(offenders)))
parentsFile = os.path.join(currentPath, "parent")
if exists_raise_eaccess(parentsFile):
parents = grabfile(parentsFile)
if not parents:
raise ParseError(_("Empty parent file: '%s'") % parentsFile)
for parentPath in parents:
abs_parent = parentPath[:1] == os.sep
if not abs_parent and allow_parent_colon:
parentPath = self._expand_parent_colon(
parentsFile, parentPath, repo_loc, repositories)
# NOTE: This os.path.join() call is intended to ignore
# currentPath if parentPath is already absolute.
parentPath = normalize_path(
os.path.join(currentPath, parentPath))
if abs_parent or repo_loc is None or \
not parentPath.startswith(repo_loc):
# It seems that this parent may point outside
# of the current repo, so realpath it.
parentPath = os.path.realpath(parentPath)
if exists_raise_eaccess(parentPath):
self._addProfile(parentPath, repositories, known_repos)
else:
raise ParseError(
_("Parent '%s' not found: '%s'") % \
(parentPath, parentsFile))
self.profiles.append(currentPath)
self.profiles_complex.append(
_profile_node(currentPath, allow_directories, False))
def _test_pty_eof():
"""
Returns True if this issues is fixed for the currently
running version of python: http://bugs.python.org/issue5380
Raises an EnvironmentError from openpty() if it fails.
"""
use_fork = False
test_string = 2 * "blah blah blah\n"
test_string = _unicode_decode(test_string,
encoding='utf_8',
errors='strict')
# may raise EnvironmentError
master_fd, slave_fd = pty.openpty()
# Non-blocking mode is required for Darwin kernel.
fcntl.fcntl(master_fd, fcntl.F_SETFL,
fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
# Disable post-processing of output since otherwise weird
# things like \n -> \r\n transformations may occur.
mode = termios.tcgetattr(slave_fd)
mode[1] &= ~termios.OPOST
termios.tcsetattr(slave_fd, termios.TCSANOW, mode)
# Simulate a subprocess writing some data to the
# slave end of the pipe, and then exiting.
pid = None
if use_fork:
pids = spawn_bash(_unicode_encode("echo -n '%s'" % test_string,
encoding='utf_8',
errors='strict'),
env=os.environ,
fd_pipes={
0: sys.stdin.fileno(),
1: slave_fd,
2: slave_fd
},
returnpid=True)
if isinstance(pids, int):
os.close(master_fd)
os.close(slave_fd)
raise EnvironmentError('spawn failed')
pid = pids[0]
else:
os.write(
slave_fd,
_unicode_encode(test_string, encoding='utf_8', errors='strict'))
os.close(slave_fd)
# If using a fork, we must wait for the child here,
# in order to avoid a race condition that would
# lead to inconsistent results.
if pid is not None:
os.waitpid(pid, 0)
master_file = os.fdopen(master_fd, 'rb')
eof = False
data = []
iwtd = [master_file]
owtd = []
ewtd = []
while not eof:
events = select.select(iwtd, owtd, ewtd)
if not events[0]:
eof = True
break
buf = array.array('B')
try:
buf.fromfile(master_file, 1024)
except (EOFError, IOError):
eof = True
if not buf:
eof = True
else:
data.append(
_unicode_decode(buf.tostring(),
encoding='utf_8',
errors='strict'))
master_file.close()
return test_string == ''.join(data)
def perform(self, qa_output):
myautoadd = self._vcs_autoadd()
self._vcs_deleted()
changes = self.get_vcs_changed()
mynew, mychanged, myremoved, no_expansion, expansion = changes
# Manifests need to be regenerated after all other commits, so don't commit
# them now even if they have changed.
mymanifests = set()
myupdates = set()
for f in mychanged + mynew:
if "Manifest" == os.path.basename(f):
mymanifests.add(f)
else:
myupdates.add(f)
myupdates.difference_update(myremoved)
myupdates = list(myupdates)
mymanifests = list(mymanifests)
myheaders = []
commitmessage = self.options.commitmsg
if self.options.commitmsgfile:
try:
f = io.open(_unicode_encode(self.options.commitmsgfile,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['content'],
errors='replace')
commitmessage = f.read()
f.close()
del f
except (IOError, OSError) as e:
if e.errno == errno.ENOENT:
portage.writemsg("!!! File Not Found:"
" --commitmsgfile='%s'\n" %
self.options.commitmsgfile)
else:
raise
if commitmessage[:9].lower() in ("cat/pkg: ", ):
commitmessage = self.msg_prefix() + commitmessage[9:]
if commitmessage is not None and commitmessage.strip():
res, expl = self.verify_commit_message(commitmessage)
if not res:
print(bad("RepoMan does not like your commit message:"))
print(expl)
if self.options.force:
print('(but proceeding due to --force)')
else:
sys.exit(1)
else:
commitmessage = None
msg_qa_output = qa_output
initial_message = None
while True:
commitmessage = self.get_new_commit_message(
msg_qa_output, commitmessage)
res, expl = self.verify_commit_message(commitmessage)
if res:
break
else:
full_expl = '''Issues with the commit message were found. Please fix it or remove
the whole commit message to abort.
''' + expl
msg_qa_output = (
[' %s\n' % x
for x in full_expl.splitlines()] + qa_output)
commitmessage = commitmessage.rstrip()
# Update copyright for new and changed files
year = time.strftime('%Y', time.gmtime())
updated_copyright = []
for fn in chain(mynew, mychanged):
if fn.endswith('.diff') or fn.endswith('.patch'):
continue
if update_copyright(fn,
year,
pretend=self.options.pretend,
owner=self.repoman_settings.get(
'COPYRIGHT_OWNER', None),
update_owner=self.options.copyright,
add_copyright=True):
updated_copyright.append(fn)
if updated_copyright and not (
self.options.pretend
or self.repo_settings.repo_config.thin_manifest):
for cp in sorted(self._vcs_files_to_cps(iter(updated_copyright))):
self._manifest_gen(cp)
myupdates, broken_changelog_manifests = self.changelogs(
myupdates, mymanifests, myremoved, mychanged, myautoadd, mynew,
commitmessage)
lines = commitmessage.splitlines()
lastline = lines[-1]
if len(lines) == 1 or re.match(r'^\S+:\s', lastline) is None:
commitmessage += '\n'
commit_footer = self.get_commit_footer()
commitmessage += commit_footer
print("* %s files being committed..." % green(str(len(myupdates))),
end=' ')
if not self.vcs_settings.needs_keyword_expansion:
# With some VCS types there's never any keyword expansion, so
# there's no need to regenerate manifests and all files will be
# committed in one big commit at the end.
logging.debug("VCS type doesn't need keyword expansion")
print()
elif not self.repo_settings.repo_config.thin_manifest:
logging.debug("perform: Calling thick_manifest()")
self.vcs_settings.changes.thick_manifest(myupdates, myheaders,
no_expansion, expansion)
logging.info("myupdates: %s", myupdates)
logging.info("myheaders: %s", myheaders)
uq = UserQuery(self.options)
if self.options.ask and uq.query('Commit changes?', True) != 'Yes':
print("* aborting commit.")
sys.exit(128 + signal.SIGINT)
# Handle the case where committed files have keywords which
# will change and need a priming commit before the Manifest
# can be committed.
if (myupdates or myremoved) and myheaders:
self.priming_commit(myupdates, myremoved, commitmessage)
# When files are removed and re-added, the cvs server will put /Attic/
# inside the $Header path. This code detects the problem and corrects it
# so that the Manifest will generate correctly. See bug #169500.
# Use binary mode in order to avoid potential character encoding issues.
self.vcs_settings.changes.clear_attic(myheaders)
if self.scanner.repolevel == 1:
utilities.repoman_sez("\"You're rather crazy... "
"doing the entire repository.\"\n")
self.vcs_settings.changes.digest_regen(myupdates, myremoved,
mymanifests, self.scanner,
broken_changelog_manifests)
if self.repo_settings.sign_manifests:
self.sign_manifest(myupdates, myremoved, mymanifests)
self.vcs_settings.changes.update_index(mymanifests, myupdates)
self.add_manifest(mymanifests, myheaders, myupdates, myremoved,
commitmessage)
if self.options.quiet:
return
print()
if self.vcs_settings.vcs:
print("Commit complete.")
else:
print("repoman was too scared"
" by not seeing any familiar version control file"
" that he forgot to commit anything")
utilities.repoman_sez(
"\"If everyone were like you, I'd be out of business!\"\n")
return
def _ebuild_exit_unlocked(self, ebuild_process, unlock_task=None):
if unlock_task is not None:
self._assert_current(unlock_task)
if unlock_task.cancelled:
self._default_final_exit(unlock_task)
return
# Normally, async_unlock should not raise an exception here.
unlock_task.future.result()
fail = False
if ebuild_process.returncode != os.EX_OK:
self.returncode = ebuild_process.returncode
if self.phase == "test" and \
"test-fail-continue" in self.settings.features:
# mark test phase as complete (bug #452030)
try:
open(_unicode_encode(os.path.join(
self.settings["PORTAGE_BUILDDIR"], ".tested"),
encoding=_encodings['fs'], errors='strict'),
'wb').close()
except OSError:
pass
else:
fail = True
if not fail:
self.returncode = None
logfile = self._get_log_path()
if self.phase == "install":
out = io.StringIO()
_check_build_log(self.settings, out=out)
msg = out.getvalue()
self.scheduler.output(msg, log_path=logfile)
if fail:
self._die_hooks()
return
settings = self.settings
_post_phase_userpriv_perms(settings)
if self.phase == "unpack":
# Bump WORKDIR timestamp, in case tar gave it a timestamp
# that will interfere with distfiles / WORKDIR timestamp
# comparisons as reported in bug #332217. Also, fix
# ownership since tar can change that too.
os.utime(settings["WORKDIR"], None)
_prepare_workdir(settings)
elif self.phase == "install":
out = io.StringIO()
_post_src_install_write_metadata(settings)
_post_src_install_uid_fix(settings, out)
msg = out.getvalue()
if msg:
self.scheduler.output(msg, log_path=logfile)
elif self.phase == "preinst":
_preinst_bsdflags(settings)
elif self.phase == "postinst":
_postinst_bsdflags(settings)
post_phase_cmds = _post_phase_cmds.get(self.phase)
if post_phase_cmds is not None:
if logfile is not None and self.phase in ("install",):
# Log to a temporary file, since the code we are running
# reads PORTAGE_LOG_FILE for QA checks, and we want to
# avoid annoying "gzip: unexpected end of file" messages
# when FEATURES=compress-build-logs is enabled.
fd, logfile = tempfile.mkstemp()
os.close(fd)
post_phase = _PostPhaseCommands(background=self.background,
commands=post_phase_cmds, elog=self._elog, fd_pipes=self.fd_pipes,
logfile=logfile, phase=self.phase, scheduler=self.scheduler,
settings=settings)
self._start_task(post_phase, self._post_phase_exit)
return
# this point is not reachable if there was a failure and
# we returned for die_hooks above, so returncode must
# indicate success (especially if ebuild_process.returncode
# is unsuccessful and test-fail-continue came into play)
self.returncode = os.EX_OK
self._current_task = None
self.wait()
def multiBuilder(self, options, settings, trees):
rValue = {}
directory = options.get(
"directory",
os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH,
"sets"))
name_pattern = options.get("name_pattern", "${name}")
if not "$name" in name_pattern and not "${name}" in name_pattern:
raise SetConfigError(
_("name_pattern doesn't include ${name} placeholder"))
greedy = get_boolean(options, "greedy", False)
# look for repository path variables
match = self._repopath_match.match(directory)
if match:
try:
directory = self._repopath_sub.sub(
trees["porttree"].dbapi.treemap[match.groupdict()
["reponame"]], directory)
except KeyError:
raise SetConfigError(
_("Could not find repository '%s'") %
match.groupdict()["reponame"])
try:
directory = _unicode_decode(directory,
encoding=_encodings['fs'],
errors='strict')
# Now verify that we can also encode it.
_unicode_encode(directory,
encoding=_encodings['fs'],
errors='strict')
except UnicodeError:
directory = _unicode_decode(directory,
encoding=_encodings['fs'],
errors='replace')
raise SetConfigError(
_("Directory path contains invalid character(s) for encoding '%s': '%s'") \
% (_encodings['fs'], directory))
if os.path.isdir(directory):
directory = normalize_path(directory)
for parent, dirs, files in os.walk(directory):
try:
parent = _unicode_decode(parent,
encoding=_encodings['fs'],
errors='strict')
except UnicodeDecodeError:
continue
for d in dirs[:]:
if d[:1] == '.':
dirs.remove(d)
for filename in files:
try:
filename = _unicode_decode(filename,
encoding=_encodings['fs'],
errors='strict')
except UnicodeDecodeError:
continue
if filename[:1] == '.':
continue
if filename.endswith(".metadata"):
continue
filename = os.path.join(parent,
filename)[1 + len(directory):]
myname = name_pattern.replace("$name", filename)
myname = myname.replace("${name}", filename)
rValue[myname] = StaticFileSet(
os.path.join(directory, filename),
greedy=greedy,
dbapi=trees["vartree"].dbapi)
return rValue
size = os.stat(filename).st_size
return (size, size)
hashfunc_map["size"] = SizeHash()
# cache all supported hash methods in a frozenset
hashfunc_keys = frozenset(hashfunc_map)
# end actual hash functions
prelink_capable = False
if os.path.exists(PRELINK_BINARY):
cmd = [PRELINK_BINARY, "--version"]
cmd = [
_unicode_encode(x, encoding=_encodings['fs'], errors='strict')
for x in cmd
]
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
proc.communicate()
status = proc.wait()
if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK:
prelink_capable = 1
del cmd, proc, status
def is_prelinkable_elf(filename):
f = _open_file(filename)
try:
def __str__(self): return _unicode_encode(self.__unicode__())
def _getitem(self, cpv):
# we override getitem because it's just a cpickling of the data handed in.
return pickle.loads(self.__db[_unicode_encode(cpv)])
def __str__(self): return _unicode_encode(self.__unicode__(), encoding=_encodings['repo.content'], errors='strict')
def _load(self, filename):
f = None
content = None
try:
f = open(_unicode_encode(filename), 'rb')
content = f.read()
except EnvironmentError as e:
if getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES):
pass
else:
writemsg(_("!!! Error loading '%s': %s\n") % \
(filename, e), noiselevel=-1)
finally:
if f is not None:
f.close()
d = None
if content:
try:
d = json.loads(_unicode_decode(content,
encoding=_encodings['repo.content'], errors='strict'))
except SystemExit:
raise
except Exception as e:
try:
mypickle = pickle.Unpickler(io.BytesIO(content))
try:
mypickle.find_global = None
except AttributeError:
# Python >=3
pass
d = mypickle.load()
except SystemExit:
raise
except Exception:
writemsg(_("!!! Error loading '%s': %s\n") % \
(filename, e), noiselevel=-1)
if d is None:
d = {}
if "old" in d:
d["updates"] = d["old"]
del d["old"]
if "cur" in d:
del d["cur"]
d.setdefault("starttime", 0)
d.setdefault("version", "")
for k in ("info", "ldpath", "updates"):
d.setdefault(k, {})
mtimedbkeys = set(("info", "ldpath", "resume", "resume_backup",
"starttime", "updates", "version"))
for k in list(d):
if k not in mtimedbkeys:
writemsg(_("Deleting invalid mtimedb key: %s\n") % str(k))
del d[k]
self.update(d)
self._clean_data = copy.deepcopy(d)
def write(self, sign=False, force=False):
"""Write Manifest instance to disk, optionally signing it. Returns
True if the Manifest is actually written, and False if the write
is skipped due to existing Manifest being identical."""
rval = False
if not self.allow_create:
return rval
self.checkIntegrity()
try:
myentries = list(self._createManifestEntries())
update_manifest = True
preserved_stats = {self.pkgdir.rstrip(os.sep): os.stat(self.pkgdir)}
if myentries and not force:
try:
with io.open(
_unicode_encode(
self.getFullname(),
encoding=_encodings["fs"],
errors="strict",
),
mode="r",
encoding=_encodings["repo.content"],
errors="replace",
) as f:
oldentries = list(self._parseManifestLines(f))
preserved_stats[self.getFullname()] = os.fstat(f.fileno())
if len(oldentries) == len(myentries):
update_manifest = False
for oldentry, myentry in zip(oldentries, myentries):
if oldentry != myentry:
update_manifest = True
break
except (IOError, OSError) as e:
if e.errno == errno.ENOENT:
pass
else:
raise
if update_manifest:
if myentries or not (self.thin or self.allow_missing):
# If myentries is empty, don't write an empty manifest
# when thin or allow_missing is enabled. Except for
# thin manifests with no DIST entries, myentries is
# non-empty for all currently known use cases.
write_atomic(
self.getFullname(),
"".join(f"{myentry}\n" for myentry in myentries),
)
self._apply_max_mtime(preserved_stats, myentries)
rval = True
else:
# With thin manifest, there's no need to have
# a Manifest file if there are no DIST entries.
try:
os.unlink(self.getFullname())
except OSError as e:
if e.errno != errno.ENOENT:
raise
rval = True
if sign:
self.sign()
except (IOError, OSError) as e:
if e.errno == errno.EACCES:
raise PermissionDenied(str(e))
raise
return rval
def updateItems(self, repoid):
"""
Figure out which news items from NEWS_PATH are both unread and relevant to
the user (according to the GLEP 42 standards of relevancy). Then add these
items into the news.repoid.unread file.
"""
# Ensure that the unread path exists and is writable.
try:
ensure_dirs(self.unread_path, uid=self._uid, gid=self._gid,
mode=self._dir_mode, mask=self._mode_mask)
except (OperationNotPermitted, PermissionDenied):
return
if not os.access(self.unread_path, os.W_OK):
return
news_dir = self._news_dir(repoid)
try:
news = _os.listdir(_unicode_encode(news_dir,
encoding=_encodings['fs'], errors='strict'))
except OSError:
return
skip_filename = self._skip_filename(repoid)
unread_filename = self._unread_filename(repoid)
unread_lock = lockfile(unread_filename, wantnewlockfile=1)
try:
try:
unread = set(grabfile(unread_filename))
unread_orig = unread.copy()
skip = set(grabfile(skip_filename))
skip_orig = skip.copy()
except PermissionDenied:
return
for itemid in news:
try:
itemid = _unicode_decode(itemid,
encoding=_encodings['fs'], errors='strict')
except UnicodeDecodeError:
itemid = _unicode_decode(itemid,
encoding=_encodings['fs'], errors='replace')
writemsg_level(
_("!!! Invalid encoding in news item name: '%s'\n") % \
itemid, level=logging.ERROR, noiselevel=-1)
continue
if itemid in skip:
continue
filename = os.path.join(news_dir, itemid,
itemid + "." + self.language_id + ".txt")
if not os.path.isfile(filename):
continue
item = NewsItem(filename, itemid)
if not item.isValid():
continue
if item.isRelevant(profile=self._profile_path,
config=self.config, vardb=self.vdb):
unread.add(item.name)
skip.add(item.name)
if unread != unread_orig:
write_atomic(unread_filename,
"".join("%s\n" % x for x in sorted(unread)))
apply_secpass_permissions(unread_filename,
uid=self._uid, gid=self._gid,
mode=self._file_mode, mask=self._mode_mask)
if skip != skip_orig:
write_atomic(skip_filename,
"".join("%s\n" % x for x in sorted(skip)))
apply_secpass_permissions(skip_filename,
uid=self._uid, gid=self._gid,
mode=self._file_mode, mask=self._mode_mask)
finally:
unlockfile(unread_lock)
def _parse_color_map(config_root='/', onerror=None):
"""
Parse /etc/portage/color.map and return a dict of error codes.
@param onerror: an optional callback to handle any ParseError that would
otherwise be raised
@type onerror: callable
@rtype: dict
@return: a dictionary mapping color classes to color codes
"""
global codes, _styles
myfile = os.path.join(config_root, COLOR_MAP_FILE)
ansi_code_pattern = re.compile("^[0-9;]*m$")
quotes = '\'"'
def strip_quotes(token):
if token[0] in quotes and token[0] == token[-1]:
token = token[1:-1]
return token
try:
lineno = 0
for line in codecs.open(_unicode_encode(myfile,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['content'],
errors='replace'):
lineno += 1
commenter_pos = line.find("#")
line = line[:commenter_pos].strip()
if len(line) == 0:
continue
split_line = line.split("=")
if len(split_line) != 2:
e = ParseError(_("'%s', line %s: expected exactly one occurence of '=' operator") % \
(myfile, lineno))
raise e
if onerror:
onerror(e)
else:
raise e
continue
k = strip_quotes(split_line[0].strip())
v = strip_quotes(split_line[1].strip())
if not k in _styles and not k in codes:
e = ParseError(_("'%s', line %s: Unknown variable: '%s'") % \
(myfile, lineno, k))
if onerror:
onerror(e)
else:
raise e
continue
if ansi_code_pattern.match(v):
if k in _styles:
_styles[k] = (esc_seq + v, )
elif k in codes:
codes[k] = esc_seq + v
else:
code_list = []
for x in v.split():
if x in codes:
if k in _styles:
code_list.append(x)
elif k in codes:
code_list.append(codes[x])
else:
e = ParseError(_("'%s', line %s: Undefined: '%s'") % \
(myfile, lineno, x))
if onerror:
onerror(e)
else:
raise e
if k in _styles:
_styles[k] = tuple(code_list)
elif k in codes:
codes[k] = "".join(code_list)
except (IOError, OSError) as e:
if e.errno == errno.ENOENT:
raise FileNotFound(myfile)
elif e.errno == errno.EACCES:
raise PermissionDenied(myfile)
raise
def _force_ascii_if_necessary(s):
# Force ascii encoding in order to avoid UnicodeEncodeError
# from smtplib.sendmail with python3 (bug #291331).
s = _unicode_encode(s, encoding="ascii", errors="backslashreplace")
s = _unicode_decode(s, encoding="ascii", errors="replace")
return s
def _calc_changelog(ebuildpath, current, next): # pylint: disable=redefined-builtin
if ebuildpath == None or not os.path.exists(ebuildpath):
return []
current = '-'.join(catpkgsplit(current)[1:])
if current.endswith('-r0'):
current = current[:-3]
next = '-'.join(catpkgsplit(next)[1:])
if next.endswith('-r0'):
next = next[:-3]
changelogdir = os.path.dirname(ebuildpath)
changelogs = ['ChangeLog']
# ChangeLog-YYYY (see bug #389611)
changelogs.extend(
sorted((fn for fn in os.listdir(changelogdir)
if fn.startswith('ChangeLog-')),
reverse=True))
divisions = []
found_current = False
for fn in changelogs:
changelogpath = os.path.join(changelogdir, fn)
try:
with io.open(_unicode_encode(changelogpath,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['repo.content'],
errors='replace') as f:
changelog = f.read()
except EnvironmentError:
return []
for node in _find_changelog_tags(changelog):
if node[0] == current:
found_current = True
break
else:
divisions.append(node)
if found_current:
break
if not found_current:
return []
#print 'XX from',current,'to',next
#for div,text in divisions: print 'XX',div
# skip entries for all revisions above the one we are about to emerge
later_rev_index = None
for i, node in enumerate(divisions):
if node[0] == next:
if later_rev_index is not None:
first_node = divisions[later_rev_index]
# Discard the later revision and the first ChangeLog entry
# that follows it. We want to display all the entries after
# that first entry, as discussed in bug #373009.
trimmed_lines = []
iterator = iter(first_node[1])
for l in iterator:
if not l:
# end of the first entry that's discarded
break
first_node = (None, list(iterator))
divisions = [first_node] + divisions[later_rev_index + 1:]
break
if node[0] is not None:
later_rev_index = i
output = []
prev_blank = False
prev_rev = False
for rev, lines in divisions:
if rev is not None:
if not (prev_blank or prev_rev):
output.append("\n")
output.append(bold('*' + rev) + '\n')
prev_rev = True
prev_blank = False
if lines:
prev_rev = False
if not prev_blank:
output.append("\n")
for l in lines:
output.append(l + "\n")
output.append("\n")
prev_blank = True
return output
def _get_unresolved_soname_deps(metadata_dir, all_provides):
"""
Get files with unresolved soname dependencies.
@param metadata_dir: directory containing package metadata files
named REQUIRES and NEEDED.ELF.2
@type metadata_dir: str
@param all_provides: a frozenset on SonameAtom instances provided by
all installed packages
@type all_provides: frozenset
@rtype: list
@return: list of tuple(filename, tuple(unresolved sonames))
"""
try:
with io.open(
_unicode_encode(
os.path.join(metadata_dir, "REQUIRES"),
encoding=_encodings["fs"],
errors="strict",
),
mode="rt",
encoding=_encodings["repo.content"],
errors="strict",
) as f:
requires = frozenset(parse_soname_deps(f.read()))
except EnvironmentError:
return []
unresolved_by_category = {}
for atom in requires:
if atom not in all_provides:
unresolved_by_category.setdefault(atom.multilib_category,
set()).add(atom.soname)
needed_filename = os.path.join(metadata_dir, "NEEDED.ELF.2")
with io.open(
_unicode_encode(needed_filename,
encoding=_encodings["fs"],
errors="strict"),
mode="rt",
encoding=_encodings["repo.content"],
errors="strict",
) as f:
needed = f.readlines()
unresolved_by_file = []
for l in needed:
l = l.rstrip("\n")
if not l:
continue
entry = NeededEntry.parse(needed_filename, l)
missing = unresolved_by_category.get(entry.multilib_category)
if not missing:
continue
# NOTE: This can contain some false positives in the case of
# missing DT_RPATH settings, since it's possible that a subset
# package files have the desired DT_RPATH settings. However,
# since reported sonames are unresolved for at least some file(s),
# false positives or this sort should not be not too annoying.
missing = [soname for soname in entry.needed if soname in missing]
if missing:
unresolved_by_file.append((entry.filename, tuple(missing)))
return unresolved_by_file
size = os.stat(filename).st_size
return (size, size)
hashfunc_map["size"] = SizeHash()
# cache all supported hash methods in a frozenset
hashfunc_keys = frozenset(hashfunc_map)
# end actual hash functions
prelink_capable = False
if os.path.exists(PRELINK_BINARY):
cmd = [PRELINK_BINARY, "--version"]
cmd = [
_unicode_encode(x, encoding=_encodings["fs"], errors="strict")
for x in cmd
]
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
proc.communicate()
status = proc.wait()
if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK:
prelink_capable = 1
del cmd, proc, status
def is_prelinkable_elf(filename):
with _open_file(filename) as f:
magic = f.read(17)
def _scan_ebuilds(self, ebuildlist, xpkg, catdir, pkgdir):
# detect unused local USE-descriptions
used_useflags = set()
for y_ebuild in ebuildlist:
ebuild = Ebuild(
self.repo_settings, self.repolevel, pkgdir, catdir, self.vcs_settings,
xpkg, y_ebuild)
if self.check['changelog'] and not self.changelog_modified \
and ebuild.ebuild_path in self.changed.new_ebuilds:
self.qatracker.add_error('changelog.ebuildadded', ebuild.relative_path)
if ebuild.untracked(self.check['ebuild_notadded'], y_ebuild, self.eadded):
# ebuild not added to vcs
self.qatracker.add_error(
"ebuild.notadded", xpkg + "/" + y_ebuild + ".ebuild")
if bad_split_check(xpkg, y_ebuild, pkgdir, self.qatracker):
continue
pkg = self.pkgs[y_ebuild]
if pkg_invalid(pkg, self.qatracker, ebuild):
self.allvalid = False
continue
myaux = pkg._metadata
eapi = myaux["EAPI"]
inherited = pkg.inherited
live_ebuild = self.live_eclasses.intersection(inherited)
self.eapicheck.check(pkg, ebuild)
for k, v in myaux.items():
if not isinstance(v, basestring):
continue
m = NON_ASCII_RE.search(v)
if m is not None:
self.qatracker.add_error(
"variable.invalidchar",
"%s: %s variable contains non-ASCII "
"character at position %s" %
(ebuild.relative_path, k, m.start() + 1))
if not self.fetchcheck.src_uri_error:
self.thirdparty.check(myaux, ebuild.relative_path)
if myaux.get("PROVIDE"):
self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
for pos, missing_var in enumerate(missingvars):
if not myaux.get(missing_var):
if catdir == "virtual" and \
missing_var in ("HOMEPAGE", "LICENSE"):
continue
if live_ebuild and missing_var == "KEYWORDS":
continue
myqakey = missingvars[pos] + ".missing"
self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild")
if catdir == "virtual":
for var in ("HOMEPAGE", "LICENSE"):
if myaux.get(var):
myqakey = var + ".virtual"
self.qatracker.add_error(myqakey, ebuild.relative_path)
self.descriptioncheck.check(pkg, ebuild)
keywords = myaux["KEYWORDS"].split()
ebuild_archs = set(
kw.lstrip("~") for kw in keywords if not kw.startswith("-"))
self.keywordcheck.check(
pkg, xpkg, ebuild, y_ebuild, keywords, ebuild_archs, self.changed,
live_ebuild, self.repo_metadata['kwlist'], self.profiles)
if live_ebuild and self.repo_settings.repo_config.name == "gentoo":
self.liveeclasscheck.check(
pkg, xpkg, ebuild, y_ebuild, keywords, self.repo_metadata['pmaskdict'])
if self.options.ignore_arches:
arches = [[
self.repo_settings.repoman_settings["ARCH"], self.repo_settings.repoman_settings["ARCH"],
self.repo_settings.repoman_settings["ACCEPT_KEYWORDS"].split()]]
else:
arches = set()
for keyword in keywords:
if keyword[0] == "-":
continue
elif keyword[0] == "~":
arch = keyword[1:]
if arch == "*":
for expanded_arch in self.profiles:
if expanded_arch == "**":
continue
arches.add(
(keyword, expanded_arch, (
expanded_arch, "~" + expanded_arch)))
else:
arches.add((keyword, arch, (arch, keyword)))
else:
# For ebuilds with stable keywords, check if the
# dependencies are satisfiable for unstable
# configurations, since use.stable.mask is not
# applied for unstable configurations (see bug
# 563546).
if keyword == "*":
for expanded_arch in self.profiles:
if expanded_arch == "**":
continue
arches.add(
(keyword, expanded_arch, (expanded_arch,)))
arches.add(
(keyword, expanded_arch,
(expanded_arch, "~" + expanded_arch)))
else:
arches.add((keyword, keyword, (keyword,)))
arches.add((keyword, keyword,
(keyword, "~" + keyword)))
if not arches:
# Use an empty profile for checking dependencies of
# packages that have empty KEYWORDS.
arches.add(('**', '**', ('**',)))
unknown_pkgs = set()
baddepsyntax = False
badlicsyntax = False
badprovsyntax = False
# catpkg = catdir + "/" + y_ebuild
inherited_java_eclass = "java-pkg-2" in inherited or \
"java-pkg-opt-2" in inherited
inherited_wxwidgets_eclass = "wxwidgets" in inherited
# operator_tokens = set(["||", "(", ")"])
type_list, badsyntax = [], []
for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
mydepstr = myaux[mytype]
buildtime = mytype in Package._buildtime_keys
runtime = mytype in Package._runtime_keys
token_class = None
if mytype.endswith("DEPEND"):
token_class = portage.dep.Atom
try:
atoms = portage.dep.use_reduce(
mydepstr, matchall=1, flat=True,
is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
except portage.exception.InvalidDependString as e:
atoms = None
badsyntax.append(str(e))
if atoms and mytype.endswith("DEPEND"):
if runtime and \
"test?" in mydepstr.split():
self.qatracker.add_error(
mytype + '.suspect',
"%s: 'test?' USE conditional in %s" %
(ebuild.relative_path, mytype))
for atom in atoms:
if atom == "||":
continue
is_blocker = atom.blocker
# Skip dependency.unknown for blockers, so that we
# don't encourage people to remove necessary blockers,
# as discussed in bug 382407. We use atom.without_use
# due to bug 525376.
if not is_blocker and \
not self.portdb.xmatch("match-all", atom.without_use) and \
not atom.cp.startswith("virtual/"):
unknown_pkgs.add((mytype, atom.unevaluated_atom))
if catdir != "virtual":
if not is_blocker and \
atom.cp in suspect_virtual:
self.qatracker.add_error(
'virtual.suspect', ebuild.relative_path +
": %s: consider using '%s' instead of '%s'" %
(mytype, suspect_virtual[atom.cp], atom))
if not is_blocker and \
atom.cp.startswith("perl-core/"):
self.qatracker.add_error('dependency.perlcore',
ebuild.relative_path +
": %s: please use '%s' instead of '%s'" %
(mytype,
atom.replace("perl-core/","virtual/perl-"),
atom))
if buildtime and \
not is_blocker and \
not inherited_java_eclass and \
atom.cp == "virtual/jdk":
self.qatracker.add_error(
'java.eclassesnotused', ebuild.relative_path)
elif buildtime and \
not is_blocker and \
not inherited_wxwidgets_eclass and \
atom.cp == "x11-libs/wxGTK":
self.qatracker.add_error(
'wxwidgets.eclassnotused',
"%s: %ss on x11-libs/wxGTK without inheriting"
" wxwidgets.eclass" % (ebuild.relative_path, mytype))
elif runtime:
if not is_blocker and \
atom.cp in suspect_rdepend:
self.qatracker.add_error(
mytype + '.suspect',
ebuild.relative_path + ": '%s'" % atom)
if atom.operator == "~" and \
portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
qacat = 'dependency.badtilde'
self.qatracker.add_error(
qacat, "%s: %s uses the ~ operator"
" with a non-zero revision: '%s'" %
(ebuild.relative_path, mytype, atom))
check_missingslot(atom, mytype, eapi, self.portdb, self.qatracker,
ebuild.relative_path, myaux)
type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
for m, b in zip(type_list, badsyntax):
if m.endswith("DEPEND"):
qacat = "dependency.syntax"
else:
qacat = m + ".syntax"
self.qatracker.add_error(
qacat, "%s: %s: %s" % (ebuild.relative_path, m, b))
badlicsyntax = len([z for z in type_list if z == "LICENSE"])
badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
badlicsyntax = badlicsyntax > 0
badprovsyntax = badprovsyntax > 0
self.use_flag_checks.check(pkg, xpkg, ebuild, y_ebuild, self.muselist)
ebuild_used_useflags = self.use_flag_checks.getUsedUseFlags()
used_useflags = used_useflags.union(ebuild_used_useflags)
self.rubyeclasscheck.check(pkg, ebuild)
# license checks
if not badlicsyntax:
self.licensecheck.check(pkg, xpkg, ebuild, y_ebuild)
self.restrictcheck.check(pkg, xpkg, ebuild, y_ebuild)
# Syntax Checks
if not self.vcs_settings.vcs_preserves_mtime:
if ebuild.ebuild_path not in self.changed.new_ebuilds and \
ebuild.ebuild_path not in self.changed.ebuilds:
pkg.mtime = None
try:
# All ebuilds should have utf_8 encoding.
f = io.open(
_unicode_encode(
ebuild.full_path, encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['repo.content'])
try:
for check_name, e in run_checks(f, pkg):
self.qatracker.add_error(
check_name, ebuild.relative_path + ': %s' % e)
finally:
f.close()
except UnicodeDecodeError:
# A file.UTF8 failure will have already been recorded above.
pass
if self.options.force:
# The dep_check() calls are the most expensive QA test. If --force
# is enabled, there's no point in wasting time on these since the
# user is intent on forcing the commit anyway.
continue
relevant_profiles = []
for keyword, arch, groups in arches:
if arch not in self.profiles:
# A missing profile will create an error further down
# during the KEYWORDS verification.
continue
if self.include_arches is not None:
if arch not in self.include_arches:
continue
relevant_profiles.extend(
(keyword, groups, prof) for prof in self.profiles[arch])
relevant_profiles.sort(key=sort_key)
for keyword, groups, prof in relevant_profiles:
is_stable_profile = prof.status == "stable"
is_dev_profile = prof.status == "dev" and \
self.options.include_dev
is_exp_profile = prof.status == "exp" and \
self.options.include_exp_profiles == 'y'
if not (is_stable_profile or is_dev_profile or is_exp_profile):
continue
dep_settings = self.caches['arch'].get(prof.sub_path)
if dep_settings is None:
dep_settings = portage.config(
config_profile_path=prof.abs_path,
config_incrementals=self.repoman_incrementals,
config_root=self.config_root,
local_config=False,
_unmatched_removal=self.options.unmatched_removal,
env=self.env, repositories=self.repo_settings.repoman_settings.repositories)
dep_settings.categories = self.repo_settings.repoman_settings.categories
if self.options.without_mask:
dep_settings._mask_manager_obj = \
copy.deepcopy(dep_settings._mask_manager)
dep_settings._mask_manager._pmaskdict.clear()
self.caches['arch'][prof.sub_path] = dep_settings
xmatch_cache_key = (prof.sub_path, tuple(groups))
xcache = self.caches['arch_xmatch'].get(xmatch_cache_key)
if xcache is None:
self.portdb.melt()
self.portdb.freeze()
xcache = self.portdb.xcache
xcache.update(self.caches['shared_xmatch'])
self.caches['arch_xmatch'][xmatch_cache_key] = xcache
self.repo_settings.trees[self.repo_settings.root]["porttree"].settings = dep_settings
self.portdb.settings = dep_settings
self.portdb.xcache = xcache
dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
# just in case, prevent config.reset() from nuking these.
dep_settings.backup_changes("ACCEPT_KEYWORDS")
# This attribute is used in dbapi._match_use() to apply
# use.stable.{mask,force} settings based on the stable
# status of the parent package. This is required in order
# for USE deps of unstable packages to be resolved correctly,
# since otherwise use.stable.{mask,force} settings of
# dependencies may conflict (see bug #456342).
dep_settings._parent_stable = dep_settings._isStable(pkg)
# Handle package.use*.{force,mask) calculation, for use
# in dep_check.
dep_settings.useforce = dep_settings._use_manager.getUseForce(
pkg, stable=dep_settings._parent_stable)
dep_settings.usemask = dep_settings._use_manager.getUseMask(
pkg, stable=dep_settings._parent_stable)
if not baddepsyntax:
ismasked = not ebuild_archs or \
pkg.cpv not in self.portdb.xmatch("match-visible",
Atom("%s::%s" % (pkg.cp, self.repo_settings.repo_config.name)))
if ismasked:
if not self.have['pmasked']:
self.have['pmasked'] = bool(dep_settings._getMaskAtom(
pkg.cpv, pkg._metadata))
if self.options.ignore_masked:
continue
# we are testing deps for a masked package; give it some lee-way
suffix = "masked"
matchmode = "minimum-all-ignore-profile"
else:
suffix = ""
matchmode = "minimum-visible"
if not self.have['dev_keywords']:
self.have['dev_keywords'] = \
bool(self.dev_keywords.intersection(keywords))
if prof.status == "dev":
suffix = suffix + "indev"
for mytype in Package._dep_keys:
mykey = "dependency.bad" + suffix
myvalue = myaux[mytype]
if not myvalue:
continue
success, atoms = portage.dep_check(
myvalue, self.portdb, dep_settings,
use="all", mode=matchmode, trees=self.repo_settings.trees)
if success:
if atoms:
# Don't bother with dependency.unknown for
# cases in which *DEPEND.bad is triggered.
for atom in atoms:
# dep_check returns all blockers and they
# aren't counted for *DEPEND.bad, so we
# ignore them here.
if not atom.blocker:
unknown_pkgs.discard(
(mytype, atom.unevaluated_atom))
if not prof.sub_path:
# old-style virtuals currently aren't
# resolvable with empty profile, since
# 'virtuals' mappings are unavailable
# (it would be expensive to search
# for PROVIDE in all ebuilds)
atoms = [
atom for atom in atoms if not (
atom.cp.startswith('virtual/')
and not self.portdb.cp_list(atom.cp))]
# we have some unsolvable deps
# remove ! deps, which always show up as unsatisfiable
atoms = [
str(atom.unevaluated_atom)
for atom in atoms if not atom.blocker]
# if we emptied out our list, continue:
if not atoms:
continue
if self.options.output_style in ['column']:
self.qatracker.add_error(mykey,
"%s: %s: %s(%s) %s"
% (ebuild.relative_path, mytype, keyword,
prof, repr(atoms)))
else:
self.qatracker.add_error(mykey,
"%s: %s: %s(%s)\n%s"
% (ebuild.relative_path, mytype, keyword,
prof, pformat(atoms, indent=6)))
else:
if self.options.output_style in ['column']:
self.qatracker.add_error(mykey,
"%s: %s: %s(%s) %s"
% (ebuild.relative_path, mytype, keyword,
prof, repr(atoms)))
else:
self.qatracker.add_error(mykey,
"%s: %s: %s(%s)\n%s"
% (ebuild.relative_path, mytype, keyword,
prof, pformat(atoms, indent=6)))
if not baddepsyntax and unknown_pkgs:
type_map = {}
for mytype, atom in unknown_pkgs:
type_map.setdefault(mytype, set()).add(atom)
for mytype, atoms in type_map.items():
self.qatracker.add_error(
"dependency.unknown", "%s: %s: %s"
% (ebuild.relative_path, mytype, ", ".join(sorted(atoms))))
# check if there are unused local USE-descriptions in metadata.xml
# (unless there are any invalids, to avoid noise)
if self.allvalid:
for myflag in self.muselist.difference(used_useflags):
self.qatracker.add_error(
"metadata.warning",
"%s/metadata.xml: unused local USE-description: '%s'"
% (xpkg, myflag))
def _start(self):
ebuild_path = self.ebuild_hash.location
with io.open(_unicode_encode(ebuild_path,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['repo.content'],
errors='replace') as f:
self._eapi, self._eapi_lineno = portage._parse_eapi_ebuild_head(f)
parsed_eapi = self._eapi
if parsed_eapi is None:
parsed_eapi = "0"
if not parsed_eapi:
# An empty EAPI setting is invalid.
self._eapi_invalid(None)
self.returncode = 1
self._async_wait()
return
self.eapi_supported = portage.eapi_is_supported(parsed_eapi)
if not self.eapi_supported:
self.metadata = {"EAPI": parsed_eapi}
self.returncode = os.EX_OK
self._async_wait()
return
settings = self.settings
settings.setcpv(self.cpv)
settings.configdict['pkg']['EAPI'] = parsed_eapi
debug = settings.get("PORTAGE_DEBUG") == "1"
master_fd = None
slave_fd = None
fd_pipes = None
if self.fd_pipes is not None:
fd_pipes = self.fd_pipes.copy()
else:
fd_pipes = {}
null_input = open('/dev/null', 'rb')
fd_pipes.setdefault(0, null_input.fileno())
fd_pipes.setdefault(1, sys.__stdout__.fileno())
fd_pipes.setdefault(2, sys.__stderr__.fileno())
# flush any pending output
stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno())
for fd in fd_pipes.values():
if fd in stdout_filenos:
sys.__stdout__.flush()
sys.__stderr__.flush()
break
self._files = self._files_dict()
files = self._files
master_fd, slave_fd = os.pipe()
fcntl.fcntl(master_fd, fcntl.F_SETFL,
fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
fd_pipes[slave_fd] = slave_fd
settings["PORTAGE_PIPE_FD"] = str(slave_fd)
self._raw_metadata = []
files.ebuild = master_fd
self.scheduler.add_reader(files.ebuild, self._output_handler)
self._registered = True
retval = portage.doebuild(ebuild_path,
"depend",
settings=settings,
debug=debug,
mydbapi=self.portdb,
tree="porttree",
fd_pipes=fd_pipes,
returnpid=True)
settings.pop("PORTAGE_PIPE_FD", None)
os.close(slave_fd)
null_input.close()
if isinstance(retval, int):
# doebuild failed before spawning
self.returncode = retval
self._async_wait()
return
self.pid = retval[0]
def _setitem(self, cpv, values):
self.__db[_unicode_encode(cpv)] = pickle.dumps(values,
pickle.HIGHEST_PROTOCOL)
def _perform_md5_merge(x, **kwargs):
return perform_md5(
_unicode_encode(x, encoding=_encodings["merge"], errors="strict"),
**kwargs)
def _perform_md5_merge(x, **kwargs):
return perform_md5(
_unicode_encode(x, encoding=_encodings['merge'], errors='strict'),
**kwargs)
def spawn(mycommand,
env=None,
opt_name=None,
fd_pipes=None,
returnpid=False,
uid=None,
gid=None,
groups=None,
umask=None,
cwd=None,
logfile=None,
path_lookup=True,
pre_exec=None,
close_fds=(sys.version_info < (3, 4)),
unshare_net=False,
unshare_ipc=False,
unshare_mount=False,
unshare_pid=False,
cgroup=None):
"""
Spawns a given command.
@param mycommand: the command to execute
@type mycommand: String or List (Popen style list)
@param env: If env is not None, it must be a mapping that defines the environment
variables for the new process; these are used instead of the default behavior
of inheriting the current process's environment.
@type env: None or Mapping
@param opt_name: an optional name for the spawn'd process (defaults to the binary name)
@type opt_name: String
@param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example
(default is {0:stdin, 1:stdout, 2:stderr})
@type fd_pipes: Dictionary
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them.
@type returnpid: Boolean
@param uid: User ID to spawn as; useful for dropping privilages
@type uid: Integer
@param gid: Group ID to spawn as; useful for dropping privilages
@type gid: Integer
@param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts.
@type groups: List
@param umask: An integer representing the umask for the process (see man chmod for umask details)
@type umask: Integer
@param cwd: Current working directory
@type cwd: String
@param logfile: name of a file to use for logging purposes
@type logfile: String
@param path_lookup: If the binary is not fully specified then look for it in PATH
@type path_lookup: Boolean
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param close_fds: If True, then close all file descriptors except those
referenced by fd_pipes (default is True for python3.3 and earlier, and False for
python3.4 and later due to non-inheritable file descriptor behavior from PEP 446).
@type close_fds: Boolean
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
@param unshare_pid: If True, PID ns will be unshared from the spawned process
@type unshare_pid: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
logfile requires stdout and stderr to be assigned to this process (ie not pointed
somewhere else.)
"""
# mycommand is either a str or a list
if isinstance(mycommand, basestring):
mycommand = mycommand.split()
env = os.environ if env is None else env
if sys.hexversion < 0x3000000:
# Avoid a potential UnicodeEncodeError from os.execve().
env_bytes = {}
for k, v in env.items():
env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \
_unicode_encode(v, encoding=_encodings['content'])
env = env_bytes
del env_bytes
# If an absolute path to an executable file isn't given
# search for it unless we've been told not to.
binary = mycommand[0]
if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \
(not os.path.isabs(binary) or not os.path.isfile(binary)
or not os.access(binary, os.X_OK)):
binary = path_lookup and find_binary(binary) or None
if not binary:
raise CommandNotFound(mycommand[0])
# If we haven't been told what file descriptors to use
# default to propagating our stdin, stdout and stderr.
if fd_pipes is None:
fd_pipes = {
0: portage._get_stdin().fileno(),
1: sys.__stdout__.fileno(),
2: sys.__stderr__.fileno(),
}
# mypids will hold the pids of all processes created.
mypids = []
if logfile:
# Using a log file requires that stdout and stderr
# are assigned to the process we're running.
if 1 not in fd_pipes or 2 not in fd_pipes:
raise ValueError(fd_pipes)
# Create a pipe
(pr, pw) = os.pipe()
# Create a tee process, giving it our stdout and stderr
# as well as the read end of the pipe.
mypids.extend(
spawn(('tee', '-i', '-a', logfile),
returnpid=True,
fd_pipes={
0: pr,
1: fd_pipes[1],
2: fd_pipes[2]
}))
# We don't need the read end of the pipe, so close it.
os.close(pr)
# Assign the write end of the pipe to our stdout and stderr.
fd_pipes[1] = pw
fd_pipes[2] = pw
# This caches the libc library lookup in the current
# process, so that it's only done once rather than
# for each child process.
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
find_library("c")
# Force instantiation of portage.data.userpriv_groups before the
# fork, so that the result is cached in the main process.
bool(groups)
parent_pid = os.getpid()
pid = None
try:
pid = os.fork()
if pid == 0:
try:
_exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups,
uid, umask, cwd, pre_exec, close_fds, unshare_net,
unshare_ipc, unshare_mount, unshare_pid, cgroup)
except SystemExit:
raise
except Exception as e:
# We need to catch _any_ exception so that it doesn't
# propagate out of this function and cause exiting
# with anything other than os._exit()
writemsg("%s:\n %s\n" % (e, " ".join(mycommand)),
noiselevel=-1)
traceback.print_exc()
sys.stderr.flush()
finally:
if pid == 0 or (pid is None and os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" % (repr(pid), ))
# Add the pid to our local and the global pid lists.
mypids.append(pid)
# If we started a tee process the write side of the pipe is no
# longer needed, so close it.
if logfile:
os.close(pw)
# If the caller wants to handle cleaning up the processes, we tell
# it about all processes that were created.
if returnpid:
return mypids
# Otherwise we clean them up.
while mypids:
# Pull the last reader in the pipe chain. If all processes
# in the pipe are well behaved, it will die when the process
# it is reading from dies.
pid = mypids.pop(0)
# and wait for it.
retval = os.waitpid(pid, 0)[1]
if retval:
# If it failed, kill off anything else that
# isn't dead yet.
for pid in mypids:
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
if os.waitpid(pid, os.WNOHANG)[0] == 0:
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0)
# If it got a signal, return the signal that was sent.
if (retval & 0xff):
return ((retval & 0xff) << 8)
# Otherwise, return its exit code.
return (retval >> 8)
# Everything succeeded
return 0
def movefile(src,
dest,
newmtime=None,
sstat=None,
mysettings=None,
hardlink_candidates=None,
encoding=_encodings['fs']):
"""moves a file from src to dest, preserving all permissions and attributes; mtime will
be preserved even when moving across filesystems. Returns mtime as integer on success
and None on failure. mtime is expressed in seconds in Python <3.3 and nanoseconds in
Python >=3.3. Move is atomic."""
if mysettings is None:
mysettings = portage.settings
src_bytes = _unicode_encode(src, encoding=encoding, errors='strict')
dest_bytes = _unicode_encode(dest, encoding=encoding, errors='strict')
xattr_enabled = "xattr" in mysettings.features
selinux_enabled = mysettings.selinux_enabled()
if selinux_enabled:
selinux = _unicode_module_wrapper(_selinux, encoding=encoding)
_copyfile = selinux.copyfile
_rename = selinux.rename
else:
_copyfile = copyfile
_rename = _os.rename
lchown = _unicode_func_wrapper(portage.data.lchown, encoding=encoding)
os = _unicode_module_wrapper(_os,
encoding=encoding,
overrides=_os_overrides)
try:
if not sstat:
sstat = os.lstat(src)
except SystemExit as e:
raise
except Exception as e:
writemsg("!!! %s\n" % _("Stating source file failed... movefile()"),
noiselevel=-1)
writemsg("!!! %s\n" % (e, ), noiselevel=-1)
return None
destexists = 1
try:
dstat = os.lstat(dest)
except (OSError, IOError):
dstat = os.lstat(os.path.dirname(dest))
destexists = 0
if bsd_chflags:
if destexists and dstat.st_flags != 0:
bsd_chflags.lchflags(dest, 0)
# Use normal stat/chflags for the parent since we want to
# follow any symlinks to the real parent directory.
pflags = os.stat(os.path.dirname(dest)).st_flags
if pflags != 0:
bsd_chflags.chflags(os.path.dirname(dest), 0)
if destexists:
if stat.S_ISLNK(dstat[stat.ST_MODE]):
try:
os.unlink(dest)
destexists = 0
except SystemExit as e:
raise
except Exception as e:
pass
if stat.S_ISLNK(sstat[stat.ST_MODE]):
try:
target = os.readlink(src)
if mysettings and "D" in mysettings and \
target.startswith(mysettings["D"]):
target = target[len(mysettings["D"]) - 1:]
if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
os.unlink(dest)
try:
if selinux_enabled:
selinux.symlink(target, dest, src)
else:
os.symlink(target, dest)
except OSError as e:
# Some programs will create symlinks automatically, so we have
# to tolerate these links being recreated during the merge
# process. In any case, if the link is pointing at the right
# place, we're in good shape.
if e.errno not in (errno.ENOENT, errno.EEXIST) or \
target != os.readlink(dest):
raise
lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID])
try:
_os.unlink(src_bytes)
except OSError:
pass
try:
os.utime(dest,
ns=(sstat.st_mtime_ns, sstat.st_mtime_ns),
follow_symlinks=False)
except NotImplementedError:
# utimensat() and lutimes() missing in libc.
return os.stat(dest, follow_symlinks=False).st_mtime_ns
else:
return sstat.st_mtime_ns
except SystemExit as e:
raise
except Exception as e:
writemsg("!!! %s\n" % _("failed to properly create symlink:"),
noiselevel=-1)
writemsg("!!! %s -> %s\n" % (dest, target), noiselevel=-1)
writemsg("!!! %s\n" % (e, ), noiselevel=-1)
return None
hardlinked = False
# Since identical files might be merged to multiple filesystems,
# so os.link() calls might fail for some paths, so try them all.
# For atomic replacement, first create the link as a temp file
# and them use os.rename() to replace the destination.
if hardlink_candidates:
head, tail = os.path.split(dest)
hardlink_tmp = os.path.join(head, ".%s._portage_merge_.%s" % \
(tail, os.getpid()))
try:
os.unlink(hardlink_tmp)
except OSError as e:
if e.errno != errno.ENOENT:
writemsg(_("!!! Failed to remove hardlink temp file: %s\n") % \
(hardlink_tmp,), noiselevel=-1)
writemsg("!!! %s\n" % (e, ), noiselevel=-1)
return None
del e
for hardlink_src in hardlink_candidates:
try:
os.link(hardlink_src, hardlink_tmp)
except OSError:
continue
else:
try:
os.rename(hardlink_tmp, dest)
except OSError as e:
writemsg(_("!!! Failed to rename %s to %s\n") % \
(hardlink_tmp, dest), noiselevel=-1)
writemsg("!!! %s\n" % (e, ), noiselevel=-1)
return None
hardlinked = True
try:
_os.unlink(src_bytes)
except OSError:
pass
break
renamefailed = 1
if hardlinked:
renamefailed = False
if not hardlinked and (selinux_enabled or sstat.st_dev == dstat.st_dev):
try:
if selinux_enabled:
selinux.rename(src, dest)
else:
os.rename(src, dest)
renamefailed = 0
except OSError as e:
if e.errno != errno.EXDEV:
# Some random error.
writemsg("!!! %s\n" % _("Failed to move %(src)s to %(dest)s") %
{
"src": src,
"dest": dest
},
noiselevel=-1)
writemsg("!!! %s\n" % (e, ), noiselevel=-1)
return None
# Invalid cross-device-link 'bind' mounted or actually Cross-Device
if renamefailed:
if stat.S_ISREG(sstat[stat.ST_MODE]):
dest_tmp = dest + "#new"
dest_tmp_bytes = _unicode_encode(dest_tmp,
encoding=encoding,
errors='strict')
try: # For safety copy then move it over.
_copyfile(src_bytes, dest_tmp_bytes)
_apply_stat(sstat, dest_tmp_bytes)
if xattr_enabled:
try:
_copyxattr(src_bytes,
dest_tmp_bytes,
exclude=mysettings.get(
"PORTAGE_XATTR_EXCLUDE", ""))
except SystemExit:
raise
except:
msg = _("Failed to copy extended attributes. "
"In order to avoid this error, set "
"FEATURES=\"-xattr\" in make.conf.")
msg = textwrap.wrap(msg, 65)
for line in msg:
writemsg("!!! %s\n" % (line, ), noiselevel=-1)
raise
_rename(dest_tmp_bytes, dest_bytes)
_os.unlink(src_bytes)
except SystemExit as e:
raise
except Exception as e:
writemsg("!!! %s\n" % _('copy %(src)s -> %(dest)s failed.') % {
"src": src,
"dest": dest
},
noiselevel=-1)
writemsg("!!! %s\n" % (e, ), noiselevel=-1)
return None
else:
#we don't yet handle special, so we need to fall back to /bin/mv
a = spawn([MOVE_BINARY, '-f', src, dest], env=os.environ)
if a != os.EX_OK:
writemsg(_("!!! Failed to move special file:\n"),
noiselevel=-1)
writemsg(_("!!! '%(src)s' to '%(dest)s'\n") % \
{"src": _unicode_decode(src, encoding=encoding),
"dest": _unicode_decode(dest, encoding=encoding)}, noiselevel=-1)
writemsg("!!! %s\n" % a, noiselevel=-1)
return None # failure
# In Python <3.3 always use stat_obj[stat.ST_MTIME] for the integral timestamp
# which is returned, since the stat_obj.st_mtime float attribute rounds *up*
# if the nanosecond part of the timestamp is 999999881 ns or greater.
try:
if hardlinked:
newmtime = os.stat(dest).st_mtime_ns
else:
# Note: It is not possible to preserve nanosecond precision
# (supported in POSIX.1-2008 via utimensat) with the IEEE 754
# double precision float which only has a 53 bit significand.
if newmtime is not None:
os.utime(dest, ns=(newmtime, newmtime))
else:
newmtime = sstat.st_mtime_ns
if renamefailed:
# If rename succeeded then timestamps are automatically
# preserved with complete precision because the source
# and destination inodes are the same. Otherwise, manually
# update timestamps with nanosecond precision.
os.utime(dest, ns=(newmtime, newmtime))
except OSError:
# The utime can fail here with EPERM even though the move succeeded.
# Instead of failing, use stat to return the mtime if possible.
try:
newmtime = os.stat(dest).st_mtime_ns
except OSError as e:
writemsg(_("!!! Failed to stat in movefile()\n"), noiselevel=-1)
writemsg("!!! %s\n" % dest, noiselevel=-1)
writemsg("!!! %s\n" % str(e), noiselevel=-1)
return None
if bsd_chflags:
# Restore the flags we saved before moving
if pflags:
bsd_chflags.chflags(os.path.dirname(dest), pflags)
return newmtime
