def __init__(self, filename, mode='w', follow_links=True, **kargs):
"""Opens a temporary filename.pid in the same directory as filename."""
ObjectProxy.__init__(self)
object.__setattr__(self, '_aborted', False)
if 'b' in mode:
open_func = open
else:
open_func = io.open
kargs.setdefault('encoding', _encodings['content'])
kargs.setdefault('errors', 'backslashreplace')
if follow_links:
canonical_path = os.path.realpath(filename)
object.__setattr__(self, '_real_name', canonical_path)
tmp_name = "%s.%i" % (canonical_path, os.getpid())
try:
object.__setattr__(self, '_file',
open_func(_unicode_encode(tmp_name,
encoding=_encodings['fs'], errors='strict'),
mode=mode, **kargs))
return
except IOError as e:
if canonical_path == filename:
raise
# Ignore this error, since it's irrelevant
# and the below open call will produce a
# new error if necessary.
object.__setattr__(self, '_real_name', filename)
tmp_name = "%s.%i" % (filename, os.getpid())
object.__setattr__(self, '_file',
open_func(_unicode_encode(tmp_name,
encoding=_encodings['fs'], errors='strict'),
mode=mode, **kargs))
def __init__(self, filename, mode='w', follow_links=True, **kargs):
"""Opens a temporary filename.pid in the same directory as filename."""
ObjectProxy.__init__(self)
object.__setattr__(self, '_aborted', False)
if 'b' in mode:
open_func = open
else:
open_func = codecs.open
kargs.setdefault('encoding', _encodings['content'])
kargs.setdefault('errors', 'backslashreplace')
if follow_links:
canonical_path = os.path.realpath(filename)
object.__setattr__(self, '_real_name', canonical_path)
tmp_name = "%s.%i" % (canonical_path, os.getpid())
try:
object.__setattr__(self, '_file',
open_func(_unicode_encode(tmp_name,
encoding=_encodings['fs'], errors='strict'),
mode=mode, **kargs))
return
except IOError as e:
if canonical_path == filename:
raise
writemsg(_("!!! Failed to open file: '%s'\n") % tmp_name,
noiselevel=-1)
writemsg("!!! %s\n" % str(e), noiselevel=-1)
object.__setattr__(self, '_real_name', filename)
tmp_name = "%s.%i" % (filename, os.getpid())
object.__setattr__(self, '_file',
open_func(_unicode_encode(tmp_name,
encoding=_encodings['fs'], errors='strict'),
mode=mode, **kargs))
def __init__(self, filename, mode='w', follow_links=True, **kargs):
"""Opens a temporary filename.pid in the same directory as filename."""
ObjectProxy.__init__(self)
object.__setattr__(self, '_aborted', False)
if 'b' in mode:
open_func = open
else:
open_func = codecs.open
kargs.setdefault('encoding', _encodings['content'])
kargs.setdefault('errors', 'backslashreplace')
if follow_links:
canonical_path = os.path.realpath(filename)
object.__setattr__(self, '_real_name', canonical_path)
tmp_name = "%s.%i" % (canonical_path, os.getpid())
try:
object.__setattr__(self, '_file',
open_func(_unicode_encode(tmp_name,
encoding=_encodings['fs'], errors='strict'),
mode=mode, **kargs))
return
except IOError as e:
if canonical_path == filename:
raise
writemsg(_("!!! Failed to open file: '%s'\n") % tmp_name,
noiselevel=-1)
writemsg("!!! %s\n" % str(e), noiselevel=-1)
object.__setattr__(self, '_real_name', filename)
tmp_name = "%s.%i" % (filename, os.getpid())
object.__setattr__(self, '_file',
open_func(_unicode_encode(tmp_name,
encoding=_encodings['fs'], errors='strict'),
mode=mode, **kargs))
def _spawn(self, args, fd_pipes=None, **kwargs): """ Fork a subprocess, apply local settings, and call fetch(). """ parent_pid = os.getpid() pid = None try: pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError( "fork returned non-integer: %s" % (repr(pid),)) return [pid] rval = 1 try: # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) # Unregister SIGCHLD handler and wakeup_fd for the parent # process's event loop (bug 655656). signal.signal(signal.SIGCHLD, signal.SIG_DFL) try: wakeup_fd = signal.set_wakeup_fd(-1) if wakeup_fd > 0: os.close(wakeup_fd) except (ValueError, OSError): pass portage.locks._close_fds() # We don't exec, so use close_fds=False # (see _setup_pipes docstring). portage.process._setup_pipes(fd_pipes, close_fds=False) rval = self._run() except SystemExit: raise except: traceback.print_exc() # os._exit() skips stderr flush! sys.stderr.flush() finally: os._exit(rval) finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1)
def recompose_mem(self, xpdata, break_hardlinks=True): """ Update the xpak segment. @param xpdata: A new xpak segment to be written, like that returned from the xpak_mem() function. @param break_hardlinks: If hardlinks exist, create a copy in order to break them. This makes it safe to use hardlinks to create cheap snapshots of the repository, which is useful for solving race conditions on binhosts as described here: http://code.google.com/p/chromium-os/issues/detail?id=3225. Default is True. """ self.scan() # Don't care about condition... We'll rewrite the data anyway. if break_hardlinks and self.filestat.st_nlink > 1: tmp_fname = "%s.%d" % (self.file, os.getpid()) shutil.copyfile(self.file, tmp_fname) try: portage.util.apply_stat_permissions(self.file, self.filestat) except portage.exception.OperationNotPermitted: pass os.rename(tmp_fname, self.file) myfile = open(_unicode_encode(self.file, encoding=_encodings['fs'], errors='strict'), 'ab+') if not myfile: raise IOError myfile.seek(-self.xpaksize,2) # 0,2 or -0,2 just mean EOF. myfile.truncate() myfile.write(xpdata+encodeint(len(xpdata)) + b'STOP') myfile.flush() myfile.close() return 1
def process(mysettings, key, logentries, fulltext):
global _items
time_str = _unicode_decode(time.strftime("%Y%m%d-%H%M%S %Z",
time.localtime(time.time())),
encoding=_encodings['content'],
errors='replace')
header = _(">>> Messages generated for package %(pkg)s by process %(pid)d on %(time)s:\n\n") % \
{"pkg": key, "pid": os.getpid(), "time": time_str}
config_root = mysettings["PORTAGE_CONFIGROOT"]
# Copy needed variables from the config instance,
# since we don't need to hold a reference for the
# whole thing. This also makes it possible to
# rely on per-package variable settings that may
# have come from /etc/portage/package.env, since
# we'll be isolated from any future mutations of
# mysettings.
config_dict = {}
for k in _config_keys:
v = mysettings.get(k)
if v is not None:
config_dict[k] = v
config_dict, items = _items.setdefault(config_root, (config_dict, {}))
items[key] = header + fulltext
def _hardlink_atomic(self, src, dest, dir_info): head, tail = os.path.split(dest) hardlink_tmp = os.path.join(head, ".%s._mirrordist_hardlink_.%s" % \ (tail, os.getpid())) try: try: os.link(src, hardlink_tmp) except OSError as e: if e.errno != errno.EXDEV: msg = "hardlink %s from %s failed: %s" % \ (self.distfile, dir_info, e) self.scheduler.output(msg + '\n', background=True, log_path=self._log_path) logging.error(msg) return False try: os.rename(hardlink_tmp, dest) except OSError as e: msg = "hardlink rename '%s' from %s failed: %s" % \ (self.distfile, dir_info, e) self.scheduler.output(msg + '\n', background=True, log_path=self._log_path) logging.error(msg) return False finally: try: os.unlink(hardlink_tmp) except OSError: pass return True
def recompose_mem(self, xpdata, break_hardlinks=True): """ Update the xpak segment. @param xpdata: A new xpak segment to be written, like that returned from the xpak_mem() function. @param break_hardlinks: If hardlinks exist, create a copy in order to break them. This makes it safe to use hardlinks to create cheap snapshots of the repository, which is useful for solving race conditions on binhosts as described here: https://crbug.com/185031 Default is True. """ self.scan() # Don't care about condition... We'll rewrite the data anyway. if break_hardlinks and self.filestat and self.filestat.st_nlink > 1: tmp_fname = "%s.%d" % (self.file, os.getpid()) shutil.copyfile(self.file, tmp_fname) try: portage.util.apply_stat_permissions(self.file, self.filestat) except portage.exception.OperationNotPermitted: pass os.rename(tmp_fname, self.file) myfile = open(_unicode_encode(self.file, encoding=_encodings['fs'], errors='strict'), 'ab+') if not myfile: raise IOError myfile.seek(-self.xpaksize, 2) # 0,2 or -0,2 just mean EOF. myfile.truncate() myfile.write(xpdata + encodeint(len(xpdata)) + b'STOP') myfile.flush() myfile.close() return 1
def _setitem(self, cpv, values):
s = cpv.rfind("/")
fp=os.path.join(self.location,cpv[:s],".update.%i.%s" % (os.getpid(), cpv[s+1:]))
try:
myf = codecs.open(_unicode_encode(fp,
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['repo.content'],
errors='backslashreplace')
except (OSError, IOError) as e:
if errno.ENOENT == e.errno:
try:
self._ensure_dirs(cpv)
myf = codecs.open(_unicode_encode(fp,
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['repo.content'],
errors='backslashreplace')
except (OSError, IOError) as e:
raise cache_errors.CacheCorruption(cpv, e)
else:
raise cache_errors.CacheCorruption(cpv, e)
for x in self.auxdbkey_order:
myf.write(values.get(x,"")+"\n")
myf.close()
self._ensure_access(fp, mtime=values["_mtime_"])
#update written. now we move it.
new_fp = os.path.join(self.location,cpv)
try:
os.rename(fp, new_fp)
except (OSError, IOError) as e:
os.remove(fp)
raise cache_errors.CacheCorruption(cpv, e)
def process(mysettings, key, logentries, fulltext):
if mysettings["PORT_LOGDIR"] != "":
elogdir = os.path.join(mysettings["PORT_LOGDIR"], "elog")
else:
elogdir = os.path.join(os.sep, "var", "log", "portage", "elog")
ensure_dirs(elogdir, uid=portage_uid, gid=portage_gid, mode=0o2770)
# TODO: Locking
elogfilename = elogdir+"/summary.log"
elogfile = codecs.open(_unicode_encode(elogfilename,
encoding=_encodings['fs'], errors='strict'),
mode='a', encoding=_encodings['content'], errors='backslashreplace')
apply_permissions(elogfilename, mode=0o60, mask=0)
time_str = time.strftime("%Y-%m-%d %H:%M:%S %Z",
time.localtime(time.time()))
# Avoid potential UnicodeDecodeError later.
time_str = _unicode_decode(time_str,
encoding=_encodings['content'], errors='replace')
elogfile.write(_(">>> Messages generated by process %(pid)d on %(time)s for package %(pkg)s:\n\n") %
{"pid": os.getpid(), "time": time_str, "pkg": key})
elogfile.write(fulltext)
elogfile.write("\n")
elogfile.close()
return elogfilename
def process(mysettings, key, logentries, fulltext):
global _items
time_str = _unicode_decode(
time.strftime("%Y%m%d-%H%M%S %Z", time.localtime(time.time())), encoding=_encodings["content"], errors="replace"
)
header = _(">>> Messages generated for package %(pkg)s by process %(pid)d on %(time)s:\n\n") % {
"pkg": key,
"pid": os.getpid(),
"time": time_str,
}
config_root = mysettings["PORTAGE_CONFIGROOT"]
# Copy needed variables from the config instance,
# since we don't need to hold a reference for the
# whole thing. This also makes it possible to
# rely on per-package variable settings that may
# have come from /etc/portage/package.env, since
# we'll be isolated from any future mutations of
# mysettings.
config_dict = {}
for k in _config_keys:
v = mysettings.get(k)
if v is not None:
config_dict[k] = v
config_dict, items = _items.setdefault(config_root, (config_dict, {}))
items[key] = header + fulltext
def process(mysettings, key, logentries, fulltext):
if mysettings["PORT_LOGDIR"] != "":
elogdir = os.path.join(mysettings["PORT_LOGDIR"], "elog")
else:
elogdir = os.path.join(os.sep, "var", "log", "portage", "elog")
ensure_dirs(elogdir, uid=portage_uid, gid=portage_gid, mode=0o2770)
# TODO: Locking
elogfilename = elogdir + "/summary.log"
elogfile = codecs.open(_unicode_encode(elogfilename,
encoding=_encodings['fs'],
errors='strict'),
mode='a',
encoding=_encodings['content'],
errors='backslashreplace')
apply_permissions(elogfilename, mode=0o60, mask=0)
time_str = time.strftime("%Y-%m-%d %H:%M:%S %Z",
time.localtime(time.time()))
# Avoid potential UnicodeDecodeError later.
time_str = _unicode_decode(time_str,
encoding=_encodings['content'],
errors='replace')
elogfile.write(
_(">>> Messages generated by process %(pid)d on %(time)s for package %(pkg)s:\n\n"
) % {
"pid": os.getpid(),
"time": time_str,
"pkg": key
})
elogfile.write(fulltext)
elogfile.write("\n")
elogfile.close()
return elogfilename
def hardlock_cleanup(path, remove_all_locks=False):
mypid = str(os.getpid())
myhost = os.uname()[1]
mydl = os.listdir(path)
results = []
mycount = 0
mylist = {}
for x in mydl:
if os.path.isfile(path + "/" + x):
parts = x.split(".hardlock-")
if len(parts) == 2:
filename = parts[0][1:]
hostpid = parts[1].split("-")
host = "-".join(hostpid[:-1])
pid = hostpid[-1]
if filename not in mylist:
mylist[filename] = {}
if host not in mylist[filename]:
mylist[filename][host] = []
mylist[filename][host].append(pid)
mycount += 1
results.append(_("Found %(count)s locks") % {"count": mycount})
for x in mylist:
if myhost in mylist[x] or remove_all_locks:
mylockname = hardlock_name(path + "/" + x)
if hardlink_is_mine(mylockname, path+"/"+x) or \
not os.path.exists(path+"/"+x) or \
remove_all_locks:
for y in mylist[x]:
for z in mylist[x][y]:
filename = path + "/." + x + ".hardlock-" + y + "-" + z
if filename == mylockname:
continue
try:
# We're sweeping through, unlinking everyone's locks.
os.unlink(filename)
results.append(_("Unlinked: ") + filename)
except OSError:
pass
try:
os.unlink(path + "/" + x)
results.append(_("Unlinked: ") + path + "/" + x)
os.unlink(mylockname)
results.append(_("Unlinked: ") + mylockname)
except OSError:
pass
else:
try:
os.unlink(mylockname)
results.append(_("Unlinked: ") + mylockname)
except OSError:
pass
return results
def _spawn(self, args, fd_pipes=None, **kwargs): """ Fork a subprocess, apply local settings, and call fetch(). """ parent_pid = os.getpid() pid = None try: pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError( "fork returned non-integer: %s" % (repr(pid),)) portage.process.spawned_pids.append(pid) return [pid] rval = 1 try: # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) portage.locks._close_fds() # We don't exec, so use close_fds=False # (see _setup_pipes docstring). portage.process._setup_pipes(fd_pipes, close_fds=False) rval = self._run() except SystemExit: raise except: traceback.print_exc() finally: os._exit(rval) finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1)
def hardlock_cleanup(path, remove_all_locks=False):
mypid = str(os.getpid())
myhost = os.uname()[1]
mydl = os.listdir(path)
results = []
mycount = 0
mylist = {}
for x in mydl:
if os.path.isfile(path + "/" + x):
parts = x.split(".hardlock-")
if len(parts) == 2:
filename = parts[0][1:]
hostpid = parts[1].split("-")
host = "-".join(hostpid[:-1])
pid = hostpid[-1]
if filename not in mylist:
mylist[filename] = {}
if host not in mylist[filename]:
mylist[filename][host] = []
mylist[filename][host].append(pid)
mycount += 1
results.append(_("Found %(count)s locks") % {"count": mycount})
for x in mylist:
if myhost in mylist[x] or remove_all_locks:
mylockname = hardlock_name(path + "/" + x)
if hardlink_is_mine(mylockname, path + "/" + x) or not os.path.exists(path + "/" + x) or remove_all_locks:
for y in mylist[x]:
for z in mylist[x][y]:
filename = path + "/." + x + ".hardlock-" + y + "-" + z
if filename == mylockname:
continue
try:
# We're sweeping through, unlinking everyone's locks.
os.unlink(filename)
results.append(_("Unlinked: ") + filename)
except OSError:
pass
try:
os.unlink(path + "/" + x)
results.append(_("Unlinked: ") + path + "/" + x)
os.unlink(mylockname)
results.append(_("Unlinked: ") + mylockname)
except OSError:
pass
else:
try:
os.unlink(mylockname)
results.append(_("Unlinked: ") + mylockname)
except OSError:
pass
return results
def process(mysettings, key, logentries, fulltext):
if mysettings.get("PORT_LOGDIR"):
logdir = normalize_path(mysettings["PORT_LOGDIR"])
else:
logdir = os.path.join(os.sep, mysettings["EPREFIX"].lstrip(os.sep),
"var", "log", "portage")
if not os.path.isdir(logdir):
# Only initialize group/mode if the directory doesn't
# exist, so that we don't override permissions if they
# were previously set by the administrator.
# NOTE: These permissions should be compatible with our
# default logrotate config as discussed in bug 374287.
logdir_uid = -1
if portage.data.secpass >= 2:
logdir_uid = portage_uid
ensure_dirs(logdir, uid=logdir_uid, gid=portage_gid, mode=0o2770)
elogdir = os.path.join(logdir, "elog")
_ensure_log_subdirs(logdir, elogdir)
# TODO: Locking
elogfilename = elogdir+"/summary.log"
elogfile = io.open(_unicode_encode(elogfilename,
encoding=_encodings['fs'], errors='strict'),
mode='a', encoding=_encodings['content'], errors='backslashreplace')
# Copy group permission bits from parent directory.
elogdir_st = os.stat(elogdir)
elogdir_gid = elogdir_st.st_gid
elogdir_grp_mode = 0o060 & elogdir_st.st_mode
# Copy the uid from the parent directory if we have privileges
# to do so, for compatibility with our default logrotate
# config (see bug 378451). With the "su portage portage"
# directive and logrotate-3.8.0, logrotate's chown call during
# the compression phase will only succeed if the log file's uid
# is portage_uid.
logfile_uid = -1
if portage.data.secpass >= 2:
logfile_uid = elogdir_st.st_uid
apply_permissions(elogfilename, uid=logfile_uid, gid=elogdir_gid,
mode=elogdir_grp_mode, mask=0)
time_str = time.strftime("%Y-%m-%d %H:%M:%S %Z",
time.localtime(time.time()))
# Avoid potential UnicodeDecodeError later.
time_str = _unicode_decode(time_str,
encoding=_encodings['content'], errors='replace')
elogfile.write(_unicode_decode(
_(">>> Messages generated by process " +
"%(pid)d on %(time)s for package %(pkg)s:\n\n") %
{"pid": os.getpid(), "time": time_str, "pkg": key}))
elogfile.write(_unicode_decode(fulltext))
elogfile.write(_unicode_decode("\n"))
elogfile.close()
return elogfilename
def process(mysettings, key, logentries, fulltext):
global _items
time_str = _unicode_decode(
time.strftime("%Y%m%d-%H%M%S %Z", time.localtime(time.time())),
encoding=_encodings['content'], errors='replace')
header = _(">>> Messages generated for package %(pkg)s by process %(pid)d on %(time)s:\n\n") % \
{"pkg": key, "pid": os.getpid(), "time": time_str}
config_root = mysettings["PORTAGE_CONFIGROOT"]
mysettings, items = _items.setdefault(config_root, (mysettings, {}))
items[key] = header + fulltext
def process(mysettings, key, logentries, fulltext):
global _items
time_str = _unicode_decode(time.strftime("%Y%m%d-%H%M%S %Z",
time.localtime(time.time())),
encoding=_encodings['content'],
errors='replace')
header = _(">>> Messages generated for package %(pkg)s by process %(pid)d on %(time)s:\n\n") % \
{"pkg": key, "pid": os.getpid(), "time": time_str}
config_root = mysettings["PORTAGE_CONFIGROOT"]
mysettings, items = _items.setdefault(config_root, (mysettings, {}))
items[key] = header + fulltext
def __init__(self, environment):
self.conf = environment
pidfile = open(self.conf.get('pid_file'), 'w')
if (self.conf.get('no_daemon') == False):
print "forking to background"
if (os.fork() == 0):
os.setpgid(0,0);
pidfile.write(str(os.getpid()));
pidfile.close();
fd = os.open("/dev/null", os.O_WRONLY);
os.dup2(fd,1);
os.close(fd);
#self.main_loop()
else:
sys.exit()
else:
print "Keeping in foreground"
pidfile.write(str(os.getpid()));
pidfile.close();
def _fetch_uri(self, uri):
if self.config.options.dry_run:
# Simply report success.
logging.info("dry-run: fetch '%s' from '%s'" %
(self.distfile, uri))
self._success()
self.returncode = os.EX_OK
self._async_wait()
return
if self.config.options.temp_dir:
self._fetch_tmp_dir_info = 'temp-dir'
distdir = self.config.options.temp_dir
else:
self._fetch_tmp_dir_info = 'distfiles'
distdir = self.config.options.distfiles
tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % os.getpid()
variables = {"DISTDIR": distdir, "URI": uri, "FILE": tmp_basename}
self._fetch_tmp_file = os.path.join(distdir, tmp_basename)
try:
os.unlink(self._fetch_tmp_file)
except OSError:
pass
args = portage.util.shlex_split(default_fetchcommand)
args = [portage.util.varexpand(x, mydict=variables) for x in args]
args = [
_unicode_encode(x, encoding=_encodings['fs'], errors='strict')
for x in args
]
null_fd = os.open(os.devnull, os.O_RDONLY)
fetcher = PopenProcess(background=self.background,
proc=subprocess.Popen(args,
stdin=null_fd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT),
scheduler=self.scheduler)
os.close(null_fd)
fetcher.pipe_reader = PipeLogger(background=self.background,
input_fd=fetcher.proc.stdout,
log_file_path=self._log_path,
scheduler=self.scheduler)
self._start_task(fetcher, self._fetcher_exit)
def _finalize(mysettings, items):
if len(items) == 0:
return
elif len(items) == 1:
count = _("one package")
else:
count = _("multiple packages")
if "PORTAGE_ELOG_MAILURI" in mysettings:
myrecipient = mysettings["PORTAGE_ELOG_MAILURI"].split()[0]
else:
myrecipient = "root@localhost"
myfrom = mysettings["PORTAGE_ELOG_MAILFROM"]
myfrom = myfrom.replace("${HOST}", socket.getfqdn())
mysubject = mysettings["PORTAGE_ELOG_MAILSUBJECT"]
mysubject = mysubject.replace("${PACKAGE}", count)
mysubject = mysubject.replace("${HOST}", socket.getfqdn())
mybody = _("elog messages for the following packages generated by "
"process %(pid)d on host %(host)s:\n") % {
"pid": os.getpid(),
"host": socket.getfqdn()
}
for key in items:
mybody += "- %s\n" % key
mymessage = portage.mail.create_message(myfrom,
myrecipient,
mysubject,
mybody,
attachments=list(items.values()))
def timeout_handler(signum, frame):
raise PortageException(
"Timeout in finalize() for elog system 'mail_summary'")
import signal
signal.signal(signal.SIGALRM, timeout_handler)
# Timeout after one minute in case send_mail() blocks indefinitely.
signal.alarm(60)
try:
try:
portage.mail.send_mail(mysettings, mymessage)
finally:
signal.alarm(0)
except PortageException as e:
writemsg("%s\n" % str(e), noiselevel=-1)
return
def _start(self): pkg = self.pkg root_config = pkg.root_config bintree = root_config.trees["bintree"] binpkg_tmpfile = os.path.join(bintree.pkgdir, pkg.cpv + ".tbz2." + str(os.getpid())) bintree._ensure_dir(os.path.dirname(binpkg_tmpfile)) self._binpkg_tmpfile = binpkg_tmpfile self.settings["PORTAGE_BINPKG_TMPFILE"] = self._binpkg_tmpfile package_phase = EbuildPhase(background=self.background, phase='package', scheduler=self.scheduler, settings=self.settings) self._start_task(package_phase, self._package_phase_exit)
def _setitem(self, cpv, values):
s = cpv.rfind("/")
fp = os.path.join(self.location, cpv[:s],
".update.%i.%s" % (os.getpid(), cpv[s + 1:]))
try:
myf = io.open(_unicode_encode(fp,
encoding=_encodings['fs'],
errors='strict'),
mode='w',
encoding=_encodings['repo.content'],
errors='backslashreplace')
except (IOError, OSError) as e:
if errno.ENOENT == e.errno:
try:
self._ensure_dirs(cpv)
myf = io.open(_unicode_encode(fp,
encoding=_encodings['fs'],
errors='strict'),
mode='w',
encoding=_encodings['repo.content'],
errors='backslashreplace')
except (OSError, IOError) as e:
raise cache_errors.CacheCorruption(cpv, e)
else:
raise cache_errors.CacheCorruption(cpv, e)
try:
for k in self._write_keys:
v = values.get(k)
if not v:
continue
# NOTE: This format string requires unicode_literals, so that
# k and v are coerced to unicode, in order to prevent TypeError
# when writing raw bytes to TextIOWrapper with Python 2.
myf.write("%s=%s\n" % (k, v))
finally:
myf.close()
self._ensure_access(fp)
#update written. now we move it.
new_fp = os.path.join(self.location, cpv)
try:
os.rename(fp, new_fp)
except (OSError, IOError) as e:
os.remove(fp)
raise cache_errors.CacheCorruption(cpv, e)
def _setitem(self, cpv, values):
s = cpv.rfind("/")
fp = os.path.join(self.location, cpv[:s], ".update.%i.%s" % (os.getpid(), cpv[s + 1 :]))
try:
myf = io.open(
_unicode_encode(fp, encoding=_encodings["fs"], errors="strict"),
mode="w",
encoding=_encodings["repo.content"],
errors="backslashreplace",
)
except (IOError, OSError) as e:
if errno.ENOENT == e.errno:
try:
self._ensure_dirs(cpv)
myf = io.open(
_unicode_encode(fp, encoding=_encodings["fs"], errors="strict"),
mode="w",
encoding=_encodings["repo.content"],
errors="backslashreplace",
)
except (OSError, IOError) as e:
raise cache_errors.CacheCorruption(cpv, e)
else:
raise cache_errors.CacheCorruption(cpv, e)
try:
for k in self._write_keys:
v = values.get(k)
if not v:
continue
# NOTE: This format string requires unicode_literals, so that
# k and v are coerced to unicode, in order to prevent TypeError
# when writing raw bytes to TextIOWrapper with Python 2.
myf.write("%s=%s\n" % (k, v))
finally:
myf.close()
self._ensure_access(fp)
# update written. now we move it.
new_fp = os.path.join(self.location, cpv)
try:
os.rename(fp, new_fp)
except (OSError, IOError) as e:
os.remove(fp)
raise cache_errors.CacheCorruption(cpv, e)
def _start(self):
pkg = self.pkg
root_config = pkg.root_config
bintree = root_config.trees["bintree"]
bintree.prevent_collision(pkg.cpv)
binpkg_tmpfile = os.path.join(bintree.pkgdir,
pkg.cpv + ".tbz2." + str(os.getpid()))
bintree._ensure_dir(os.path.dirname(binpkg_tmpfile))
self._binpkg_tmpfile = binpkg_tmpfile
self.settings["PORTAGE_BINPKG_TMPFILE"] = self._binpkg_tmpfile
package_phase = EbuildPhase(background=self.background,
phase='package',
scheduler=self.scheduler,
settings=self.settings)
self._start_task(package_phase, self._package_phase_exit)
def _setitem(self, cpv, values):
# import pdb;pdb.set_trace()
s = cpv.rfind("/")
fp = os.path.join(self.location, cpv[:s],
".update.%i.%s" % (os.getpid(), cpv[s + 1:]))
try:
myf = codecs.open(_unicode_encode(fp,
encoding=_encodings['fs'],
errors='strict'),
mode='w',
encoding=_encodings['repo.content'],
errors='backslashreplace')
except (IOError, OSError) as e:
if errno.ENOENT == e.errno:
try:
self._ensure_dirs(cpv)
myf = codecs.open(_unicode_encode(
fp, encoding=_encodings['fs'], errors='strict'),
mode='w',
encoding=_encodings['repo.content'],
errors='backslashreplace')
except (OSError, IOError) as e:
raise cache_errors.CacheCorruption(cpv, e)
else:
raise cache_errors.CacheCorruption(cpv, e)
try:
for k in self._write_keys:
v = values.get(k)
if not v:
continue
myf.write("%s=%s\n" % (k, v))
finally:
myf.close()
self._ensure_access(fp)
#update written. now we move it.
new_fp = os.path.join(self.location, cpv)
try:
os.rename(fp, new_fp)
except (OSError, IOError) as e:
os.remove(fp)
raise cache_errors.CacheCorruption(cpv, e)
def _finalize(mysettings, items):
if len(items) == 0:
return
elif len(items) == 1:
count = _("one package")
else:
count = _("multiple packages")
if "PORTAGE_ELOG_MAILURI" in mysettings:
myrecipient = mysettings["PORTAGE_ELOG_MAILURI"].split()[0]
else:
myrecipient = "root@localhost"
myfrom = mysettings["PORTAGE_ELOG_MAILFROM"]
myfrom = myfrom.replace("${HOST}", socket.getfqdn())
mysubject = mysettings["PORTAGE_ELOG_MAILSUBJECT"]
mysubject = mysubject.replace("${PACKAGE}", count)
mysubject = mysubject.replace("${HOST}", socket.getfqdn())
mybody = _("elog messages for the following packages generated by "
"process %(pid)d on host %(host)s:\n") % {"pid": os.getpid(), "host": socket.getfqdn()}
for key in items:
mybody += "- %s\n" % key
mymessage = portage.mail.create_message(myfrom, myrecipient, mysubject,
mybody, attachments=list(items.values()))
def timeout_handler(signum, frame):
raise PortageException("Timeout in finalize() for elog system 'mail_summary'")
import signal
signal.signal(signal.SIGALRM, timeout_handler)
# Timeout after one minute in case send_mail() blocks indefinitely.
signal.alarm(60)
try:
try:
portage.mail.send_mail(mysettings, mymessage)
finally:
signal.alarm(0)
except PortageException as e:
writemsg("%s\n" % str(e), noiselevel=-1)
return
def _setitem(self, cpv, values):
# import pdb;pdb.set_trace()
s = cpv.rfind("/")
fp = os.path.join(self.location,cpv[:s],".update.%i.%s" % (os.getpid(), cpv[s+1:]))
try:
myf = codecs.open(_unicode_encode(fp,
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['repo.content'],
errors='backslashreplace')
except (IOError, OSError) as e:
if errno.ENOENT == e.errno:
try:
self._ensure_dirs(cpv)
myf = codecs.open(_unicode_encode(fp,
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['repo.content'],
errors='backslashreplace')
except (OSError, IOError) as e:
raise cache_errors.CacheCorruption(cpv, e)
else:
raise cache_errors.CacheCorruption(cpv, e)
try:
for k in self._write_keys:
v = values.get(k)
if not v:
continue
myf.write("%s=%s\n" % (k, v))
finally:
myf.close()
self._ensure_access(fp)
#update written. now we move it.
new_fp = os.path.join(self.location,cpv)
try:
os.rename(fp, new_fp)
except (OSError, IOError) as e:
os.remove(fp)
raise cache_errors.CacheCorruption(cpv, e)
def _hardlink_atomic(self, src, dest, dir_info, symlink=False):
head, tail = os.path.split(dest)
hardlink_tmp = os.path.join(head, ".%s._mirrordist_hardlink_.%s" % \
(tail, os.getpid()))
try:
try:
if symlink:
os.symlink(src, hardlink_tmp)
else:
os.link(src, hardlink_tmp)
except OSError as e:
if e.errno != errno.EXDEV:
msg = "hardlink %s from %s failed: %s" % \
(self.distfile, dir_info, e)
self.scheduler.output(msg + '\n',
background=True,
log_path=self._log_path)
logging.error(msg)
return False
try:
os.rename(hardlink_tmp, dest)
except OSError as e:
msg = "hardlink rename '%s' from %s failed: %s" % \
(self.distfile, dir_info, e)
self.scheduler.output(msg + '\n',
background=True,
log_path=self._log_path)
logging.error(msg)
return False
finally:
try:
os.unlink(hardlink_tmp)
except OSError:
pass
return True
def ionice(settings):
ionice_cmd = settings.get("PORTAGE_IONICE_COMMAND")
if ionice_cmd:
ionice_cmd = portage.util.shlex_split(ionice_cmd)
if not ionice_cmd:
return
from portage.util import varexpand
variables = {"PID" : str(os.getpid())}
cmd = [varexpand(x, mydict=variables) for x in ionice_cmd]
try:
rval = portage.process.spawn(cmd, env=os.environ)
except portage.exception.CommandNotFound:
# The OS kernel probably doesn't support ionice,
# so return silently.
return
if rval != os.EX_OK:
out = portage.output.EOutput()
out.eerror("PORTAGE_IONICE_COMMAND returned %d" % (rval,))
out.eerror("See the make.conf(5) man page for PORTAGE_IONICE_COMMAND usage instructions.")
def _start(self):
self.phase = "package"
self.tree = "porttree"
pkg = self.pkg
root_config = pkg.root_config
portdb = root_config.trees["porttree"].dbapi
bintree = root_config.trees["bintree"]
ebuild_path = portdb.findname(pkg.cpv)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % pkg.cpv)
settings = self.settings
debug = settings.get("PORTAGE_DEBUG") == "1"
bintree.prevent_collision(pkg.cpv)
binpkg_tmpfile = os.path.join(bintree.pkgdir,
pkg.cpv + ".tbz2." + str(os.getpid()))
self._binpkg_tmpfile = binpkg_tmpfile
settings["PORTAGE_BINPKG_TMPFILE"] = binpkg_tmpfile
settings.backup_changes("PORTAGE_BINPKG_TMPFILE")
try:
EbuildProcess._start(self)
finally:
settings.pop("PORTAGE_BINPKG_TMPFILE", None)
'portage.util:dump_traceback',
)
from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY
from portage.exception import CommandNotFound
try:
import resource
max_fd_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
except ImportError:
max_fd_limit = 256
if sys.hexversion >= 0x3000000:
basestring = str
if os.path.isdir("/proc/%i/fd" % os.getpid()):
def get_open_fds():
return (int(fd) for fd in os.listdir("/proc/%i/fd" % os.getpid()) \
if fd.isdigit())
if platform.python_implementation() == 'PyPy':
# EAGAIN observed with PyPy 1.8.
_get_open_fds = get_open_fds
def get_open_fds():
try:
return _get_open_fds()
except OSError as e:
if e.errno != errno.EAGAIN:
raise
return range(max_fd_limit)
def hardlock_name(path): base, tail = os.path.split(path) return os.path.join(base, ".%s.hardlock-%s-%s" % (tail, os.uname()[1], os.getpid()))
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
pre_exec, close_fds, unshare_net, unshare_ipc, cgroup):
"""
Execute a given binary with options
@param binary: Name of program to execute
@type binary: String
@param mycommand: Options for program
@type mycommand: String
@param opt_name: Name of process (defaults to binary)
@type opt_name: String
@param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 }
@type fd_pipes: Dictionary
@param env: Key,Value mapping for Environmental Variables
@type env: Dictionary
@param gid: Group ID to run the process under
@type gid: Integer
@param groups: Groups the Process should be in.
@type groups: Integer
@param uid: User ID to run the process under
@type uid: Integer
@param umask: an int representing a unix umask (see man chmod for umask details)
@type umask: Integer
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
@rtype: None
@return: Never returns (calls os.execve)
"""
# If the process we're creating hasn't been given a name
# assign it the name of the executable.
if not opt_name:
if binary is portage._python_interpreter:
# NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0]
# does not contain the full path of the binary.
opt_name = binary
else:
opt_name = os.path.basename(binary)
# Set up the command's argument list.
myargs = [opt_name]
myargs.extend(mycommand[1:])
# Avoid a potential UnicodeEncodeError from os.execve().
myargs = [
_unicode_encode(x, encoding=_encodings['fs'], errors='strict')
for x in myargs
]
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# Quiet killing of subprocesses by SIGPIPE (see bug #309001).
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
# Avoid issues triggered by inheritance of SIGQUIT handler from
# the parent process (see bug #289486).
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
_setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
# Add to cgroup
# it's better to do it from the child since we can guarantee
# it is done before we start forking children
if cgroup:
with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f:
f.write('%d\n' % os.getpid())
# Unshare (while still uid==0)
if unshare_net or unshare_ipc:
filename = find_library("c")
if filename is not None:
libc = LoadLibrary(filename)
if libc is not None:
CLONE_NEWIPC = 0x08000000
CLONE_NEWNET = 0x40000000
flags = 0
if unshare_net:
flags |= CLONE_NEWNET
if unshare_ipc:
flags |= CLONE_NEWIPC
try:
if libc.unshare(flags) != 0:
writemsg(
"Unable to unshare: %s\n" %
(errno.errorcode.get(ctypes.get_errno(), '?')),
noiselevel=-1)
else:
if unshare_net:
# 'up' the loopback
IFF_UP = 0x1
ifreq = struct.pack('16sh', b'lo', IFF_UP)
SIOCSIFFLAGS = 0x8914
sock = socket.socket(socket.AF_INET,
socket.SOCK_DGRAM, 0)
try:
fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq)
except IOError as e:
writemsg(
"Unable to enable loopback interface: %s\n"
% (errno.errorcode.get(e.errno, '?')),
noiselevel=-1)
sock.close()
except AttributeError:
# unshare() not supported by libc
pass
# Set requested process permissions.
if gid:
# Cast proxies to int, in case it matters.
os.setgid(int(gid))
if groups:
os.setgroups(groups)
if uid:
# Cast proxies to int, in case it matters.
os.setuid(int(uid))
if umask:
os.umask(umask)
if pre_exec:
pre_exec()
# And switch to the new process.
os.execve(binary, myargs, env)
def spawn(mycommand,
env={},
opt_name=None,
fd_pipes=None,
returnpid=False,
uid=None,
gid=None,
groups=None,
umask=None,
logfile=None,
path_lookup=True,
pre_exec=None,
close_fds=(sys.version_info < (3, 4)),
unshare_net=False,
unshare_ipc=False,
cgroup=None):
"""
Spawns a given command.
@param mycommand: the command to execute
@type mycommand: String or List (Popen style list)
@param env: A dict of Key=Value pairs for env variables
@type env: Dictionary
@param opt_name: an optional name for the spawn'd process (defaults to the binary name)
@type opt_name: String
@param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example
(default is {0:stdin, 1:stdout, 2:stderr})
@type fd_pipes: Dictionary
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them.
@type returnpid: Boolean
@param uid: User ID to spawn as; useful for dropping privilages
@type uid: Integer
@param gid: Group ID to spawn as; useful for dropping privilages
@type gid: Integer
@param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts.
@type groups: List
@param umask: An integer representing the umask for the process (see man chmod for umask details)
@type umask: Integer
@param logfile: name of a file to use for logging purposes
@type logfile: String
@param path_lookup: If the binary is not fully specified then look for it in PATH
@type path_lookup: Boolean
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param close_fds: If True, then close all file descriptors except those
referenced by fd_pipes (default is True for python3.3 and earlier, and False for
python3.4 and later due to non-inheritable file descriptor behavior from PEP 446).
@type close_fds: Boolean
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
logfile requires stdout and stderr to be assigned to this process (ie not pointed
somewhere else.)
"""
# mycommand is either a str or a list
if isinstance(mycommand, basestring):
mycommand = mycommand.split()
if sys.hexversion < 0x3000000:
# Avoid a potential UnicodeEncodeError from os.execve().
env_bytes = {}
for k, v in env.items():
env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \
_unicode_encode(v, encoding=_encodings['content'])
env = env_bytes
del env_bytes
# If an absolute path to an executable file isn't given
# search for it unless we've been told not to.
binary = mycommand[0]
if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \
(not os.path.isabs(binary) or not os.path.isfile(binary)
or not os.access(binary, os.X_OK)):
binary = path_lookup and find_binary(binary) or None
if not binary:
raise CommandNotFound(mycommand[0])
# If we haven't been told what file descriptors to use
# default to propagating our stdin, stdout and stderr.
if fd_pipes is None:
fd_pipes = {
0: portage._get_stdin().fileno(),
1: sys.__stdout__.fileno(),
2: sys.__stderr__.fileno(),
}
# mypids will hold the pids of all processes created.
mypids = []
if logfile:
# Using a log file requires that stdout and stderr
# are assigned to the process we're running.
if 1 not in fd_pipes or 2 not in fd_pipes:
raise ValueError(fd_pipes)
# Create a pipe
(pr, pw) = os.pipe()
# Create a tee process, giving it our stdout and stderr
# as well as the read end of the pipe.
mypids.extend(
spawn(('tee', '-i', '-a', logfile),
returnpid=True,
fd_pipes={
0: pr,
1: fd_pipes[1],
2: fd_pipes[2]
}))
# We don't need the read end of the pipe, so close it.
os.close(pr)
# Assign the write end of the pipe to our stdout and stderr.
fd_pipes[1] = pw
fd_pipes[2] = pw
# This caches the libc library lookup in the current
# process, so that it's only done once rather than
# for each child process.
if unshare_net or unshare_ipc:
find_library("c")
# Force instantiation of portage.data.userpriv_groups before the
# fork, so that the result is cached in the main process.
bool(groups)
parent_pid = os.getpid()
pid = None
try:
pid = os.fork()
if pid == 0:
try:
_exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups,
uid, umask, pre_exec, close_fds, unshare_net,
unshare_ipc, cgroup)
except SystemExit:
raise
except Exception as e:
# We need to catch _any_ exception so that it doesn't
# propagate out of this function and cause exiting
# with anything other than os._exit()
writemsg("%s:\n %s\n" % (e, " ".join(mycommand)),
noiselevel=-1)
traceback.print_exc()
sys.stderr.flush()
finally:
if pid == 0 or (pid is None and os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" % (repr(pid), ))
# Add the pid to our local and the global pid lists.
mypids.append(pid)
# If we started a tee process the write side of the pipe is no
# longer needed, so close it.
if logfile:
os.close(pw)
# If the caller wants to handle cleaning up the processes, we tell
# it about all processes that were created.
if returnpid:
return mypids
# Otherwise we clean them up.
while mypids:
# Pull the last reader in the pipe chain. If all processes
# in the pipe are well behaved, it will die when the process
# it is reading from dies.
pid = mypids.pop(0)
# and wait for it.
retval = os.waitpid(pid, 0)[1]
if retval:
# If it failed, kill off anything else that
# isn't dead yet.
for pid in mypids:
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
if os.waitpid(pid, os.WNOHANG)[0] == 0:
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0)
# If it got a signal, return the signal that was sent.
if (retval & 0xff):
return ((retval & 0xff) << 8)
# Otherwise, return its exit code.
return (retval >> 8)
# Everything succeeded
return 0
def hardlock_name(path):
base, tail = os.path.split(path)
return os.path.join(
base, ".%s.hardlock-%s-%s" %
(tail, portage._decode_argv([os.uname()[1]])[0], os.getpid()))
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
pre_exec, close_fds, unshare_net, unshare_ipc, cgroup):
"""
Execute a given binary with options
@param binary: Name of program to execute
@type binary: String
@param mycommand: Options for program
@type mycommand: String
@param opt_name: Name of process (defaults to binary)
@type opt_name: String
@param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 }
@type fd_pipes: Dictionary
@param env: Key,Value mapping for Environmental Variables
@type env: Dictionary
@param gid: Group ID to run the process under
@type gid: Integer
@param groups: Groups the Process should be in.
@type groups: Integer
@param uid: User ID to run the process under
@type uid: Integer
@param umask: an int representing a unix umask (see man chmod for umask details)
@type umask: Integer
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
@rtype: None
@return: Never returns (calls os.execve)
"""
# If the process we're creating hasn't been given a name
# assign it the name of the executable.
if not opt_name:
if binary is portage._python_interpreter:
# NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0]
# does not contain the full path of the binary.
opt_name = binary
else:
opt_name = os.path.basename(binary)
# Set up the command's argument list.
myargs = [opt_name]
myargs.extend(mycommand[1:])
# Avoid a potential UnicodeEncodeError from os.execve().
myargs = [_unicode_encode(x, encoding=_encodings['fs'],
errors='strict') for x in myargs]
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# Quiet killing of subprocesses by SIGPIPE (see bug #309001).
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
# Avoid issues triggered by inheritance of SIGQUIT handler from
# the parent process (see bug #289486).
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
_setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
# Add to cgroup
# it's better to do it from the child since we can guarantee
# it is done before we start forking children
if cgroup:
with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f:
f.write('%d\n' % os.getpid())
# Unshare (while still uid==0)
if unshare_net or unshare_ipc:
filename = find_library("c")
if filename is not None:
libc = LoadLibrary(filename)
if libc is not None:
CLONE_NEWIPC = 0x08000000
CLONE_NEWNET = 0x40000000
flags = 0
if unshare_net:
flags |= CLONE_NEWNET
if unshare_ipc:
flags |= CLONE_NEWIPC
try:
if libc.unshare(flags) != 0:
writemsg("Unable to unshare: %s\n" % (
errno.errorcode.get(ctypes.get_errno(), '?')),
noiselevel=-1)
else:
if unshare_net:
# 'up' the loopback
IFF_UP = 0x1
ifreq = struct.pack('16sh', b'lo', IFF_UP)
SIOCSIFFLAGS = 0x8914
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
try:
fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq)
except IOError as e:
writemsg("Unable to enable loopback interface: %s\n" % (
errno.errorcode.get(e.errno, '?')),
noiselevel=-1)
sock.close()
except AttributeError:
# unshare() not supported by libc
pass
# Set requested process permissions.
if gid:
# Cast proxies to int, in case it matters.
os.setgid(int(gid))
if groups:
os.setgroups(groups)
if uid:
# Cast proxies to int, in case it matters.
os.setuid(int(uid))
if umask:
os.umask(umask)
if pre_exec:
pre_exec()
# And switch to the new process.
os.execve(binary, myargs, env)
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
cwd, pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount,
unshare_pid, cgroup):
"""
Execute a given binary with options
@param binary: Name of program to execute
@type binary: String
@param mycommand: Options for program
@type mycommand: String
@param opt_name: Name of process (defaults to binary)
@type opt_name: String
@param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 }
@type fd_pipes: Dictionary
@param env: Key,Value mapping for Environmental Variables
@type env: Dictionary
@param gid: Group ID to run the process under
@type gid: Integer
@param groups: Groups the Process should be in.
@type groups: Integer
@param uid: User ID to run the process under
@type uid: Integer
@param umask: an int representing a unix umask (see man chmod for umask details)
@type umask: Integer
@param cwd: Current working directory
@type cwd: String
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
@param unshare_pid: If True, PID ns will be unshared from the spawned process
@type unshare_pid: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
@rtype: None
@return: Never returns (calls os.execve)
"""
# If the process we're creating hasn't been given a name
# assign it the name of the executable.
if not opt_name:
if binary is portage._python_interpreter:
# NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0]
# does not contain the full path of the binary.
opt_name = binary
else:
opt_name = os.path.basename(binary)
# Set up the command's argument list.
myargs = [opt_name]
myargs.extend(mycommand[1:])
# Avoid a potential UnicodeEncodeError from os.execve().
myargs = [
_unicode_encode(x, encoding=_encodings['fs'], errors='strict')
for x in myargs
]
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# Unregister SIGCHLD handler and wakeup_fd for the parent
# process's event loop (bug 655656).
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
try:
wakeup_fd = signal.set_wakeup_fd(-1)
if wakeup_fd > 0:
os.close(wakeup_fd)
except (ValueError, OSError):
pass
# Quiet killing of subprocesses by SIGPIPE (see bug #309001).
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
# Avoid issues triggered by inheritance of SIGQUIT handler from
# the parent process (see bug #289486).
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
_setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
# Add to cgroup
# it's better to do it from the child since we can guarantee
# it is done before we start forking children
if cgroup:
with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f:
f.write('%d\n' % os.getpid())
# Unshare (while still uid==0)
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
filename = find_library("c")
if filename is not None:
libc = LoadLibrary(filename)
if libc is not None:
# from /usr/include/bits/sched.h
CLONE_NEWNS = 0x00020000
CLONE_NEWIPC = 0x08000000
CLONE_NEWPID = 0x20000000
CLONE_NEWNET = 0x40000000
flags = 0
if unshare_net:
flags |= CLONE_NEWNET
if unshare_ipc:
flags |= CLONE_NEWIPC
if unshare_mount:
# NEWNS = mount namespace
flags |= CLONE_NEWNS
if unshare_pid:
# we also need mount namespace for slave /proc
flags |= CLONE_NEWPID | CLONE_NEWNS
try:
if libc.unshare(flags) != 0:
writemsg(
"Unable to unshare: %s\n" %
(errno.errorcode.get(ctypes.get_errno(), '?')),
noiselevel=-1)
else:
if unshare_pid:
# pid namespace requires us to become init
fork_ret = os.fork()
if fork_ret != 0:
os.execv(portage._python_interpreter, [
portage._python_interpreter,
os.path.join(portage._bin_path,
'pid-ns-init'),
'%s' % fork_ret,
])
if unshare_mount:
# mark the whole filesystem as slave to avoid
# mounts escaping the namespace
s = subprocess.Popen(
['mount', '--make-rslave', '/'])
mount_ret = s.wait()
if mount_ret != 0:
# TODO: should it be fatal maybe?
writemsg("Unable to mark mounts slave: %d\n" %
(mount_ret, ),
noiselevel=-1)
if unshare_pid:
# we need at least /proc being slave
s = subprocess.Popen(
['mount', '--make-slave', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
# can't proceed with shared /proc
writemsg("Unable to mark /proc slave: %d\n" %
(mount_ret, ),
noiselevel=-1)
os._exit(1)
# mount new /proc for our namespace
s = subprocess.Popen(
['mount', '-t', 'proc', 'proc', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
writemsg("Unable to mount new /proc: %d\n" %
(mount_ret, ),
noiselevel=-1)
os._exit(1)
if unshare_net:
# 'up' the loopback
IFF_UP = 0x1
ifreq = struct.pack('16sh', b'lo', IFF_UP)
SIOCSIFFLAGS = 0x8914
sock = socket.socket(socket.AF_INET,
socket.SOCK_DGRAM, 0)
try:
fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq)
except IOError as e:
writemsg(
"Unable to enable loopback interface: %s\n"
% (errno.errorcode.get(e.errno, '?')),
noiselevel=-1)
sock.close()
except AttributeError:
# unshare() not supported by libc
pass
# Set requested process permissions.
if gid:
# Cast proxies to int, in case it matters.
os.setgid(int(gid))
if groups:
os.setgroups(groups)
if uid:
# Cast proxies to int, in case it matters.
os.setuid(int(uid))
if umask:
os.umask(umask)
if cwd is not None:
os.chdir(cwd)
if pre_exec:
pre_exec()
# And switch to the new process.
os.execve(binary, myargs, env)
def _exec(binary, mycommand, opt_name, fd_pipes,
env, gid, groups, uid, umask, cwd,
pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid,
unshare_flags, cgroup):
"""
Execute a given binary with options
@param binary: Name of program to execute
@type binary: String
@param mycommand: Options for program
@type mycommand: String
@param opt_name: Name of process (defaults to binary)
@type opt_name: String
@param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 }
@type fd_pipes: Dictionary
@param env: Key,Value mapping for Environmental Variables
@type env: Dictionary
@param gid: Group ID to run the process under
@type gid: Integer
@param groups: Groups the Process should be in.
@type groups: List
@param uid: User ID to run the process under
@type uid: Integer
@param umask: an int representing a unix umask (see man chmod for umask details)
@type umask: Integer
@param cwd: Current working directory
@type cwd: String
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
@param unshare_pid: If True, PID ns will be unshared from the spawned process
@type unshare_pid: Boolean
@param unshare_flags: Flags for the unshare(2) function
@type unshare_flags: Integer
@param cgroup: CGroup path to bind the process to
@type cgroup: String
@rtype: None
@return: Never returns (calls os.execve)
"""
# If the process we're creating hasn't been given a name
# assign it the name of the executable.
if not opt_name:
if binary is portage._python_interpreter:
# NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0]
# does not contain the full path of the binary.
opt_name = binary
else:
opt_name = os.path.basename(binary)
# Set up the command's argument list.
myargs = [opt_name]
myargs.extend(mycommand[1:])
# Avoid a potential UnicodeEncodeError from os.execve().
myargs = [_unicode_encode(x, encoding=_encodings['fs'],
errors='strict') for x in myargs]
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# Unregister SIGCHLD handler and wakeup_fd for the parent
# process's event loop (bug 655656).
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
try:
wakeup_fd = signal.set_wakeup_fd(-1)
if wakeup_fd > 0:
os.close(wakeup_fd)
except (ValueError, OSError):
pass
# Quiet killing of subprocesses by SIGPIPE (see bug #309001).
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
# Avoid issues triggered by inheritance of SIGQUIT handler from
# the parent process (see bug #289486).
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
_setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
# Add to cgroup
# it's better to do it from the child since we can guarantee
# it is done before we start forking children
if cgroup:
with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f:
f.write('%d\n' % os.getpid())
# Unshare (while still uid==0)
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
filename = find_library("c")
if filename is not None:
libc = LoadLibrary(filename)
if libc is not None:
try:
# Since a failed unshare call could corrupt process
# state, first validate that the call can succeed.
# The parent process should call _unshare_validate
# before it forks, so that all child processes can
# reuse _unshare_validate results that have been
# cached by the parent process.
errno_value = _unshare_validate(unshare_flags)
if errno_value == 0 and libc.unshare(unshare_flags) != 0:
errno_value = ctypes.get_errno()
if errno_value != 0:
writemsg("Unable to unshare: %s\n" % (
errno.errorcode.get(errno_value, '?')),
noiselevel=-1)
else:
if unshare_pid:
main_child_pid = os.fork()
if main_child_pid == 0:
# pid namespace requires us to become init
binary, myargs = portage._python_interpreter, [
portage._python_interpreter,
os.path.join(portage._bin_path,
'pid-ns-init'),
_unicode_encode('' if uid is None else str(uid)),
_unicode_encode('' if gid is None else str(gid)),
_unicode_encode('' if groups is None else ','.join(str(group) for group in groups)),
_unicode_encode('' if umask is None else str(umask)),
_unicode_encode(','.join(str(fd) for fd in fd_pipes)),
binary] + myargs
uid = None
gid = None
groups = None
umask = None
else:
# Execute a supervisor process which will forward
# signals to init and forward exit status to the
# parent process. The supervisor process runs in
# the global pid namespace, so skip /proc remount
# and other setup that's intended only for the
# init process.
binary, myargs = portage._python_interpreter, [
portage._python_interpreter,
os.path.join(portage._bin_path,
'pid-ns-init'), str(main_child_pid)]
os.execve(binary, myargs, env)
if unshare_mount:
# mark the whole filesystem as slave to avoid
# mounts escaping the namespace
s = subprocess.Popen(['mount',
'--make-rslave', '/'])
mount_ret = s.wait()
if mount_ret != 0:
# TODO: should it be fatal maybe?
writemsg("Unable to mark mounts slave: %d\n" % (mount_ret,),
noiselevel=-1)
if unshare_pid:
# we need at least /proc being slave
s = subprocess.Popen(['mount',
'--make-slave', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
# can't proceed with shared /proc
writemsg("Unable to mark /proc slave: %d\n" % (mount_ret,),
noiselevel=-1)
os._exit(1)
# mount new /proc for our namespace
s = subprocess.Popen(['mount',
'-n', '-t', 'proc', 'proc', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
writemsg("Unable to mount new /proc: %d\n" % (mount_ret,),
noiselevel=-1)
os._exit(1)
if unshare_net:
# 'up' the loopback
IFF_UP = 0x1
ifreq = struct.pack('16sh', b'lo', IFF_UP)
SIOCSIFFLAGS = 0x8914
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
try:
fcntl.ioctl(sock, SIOCSIFFLAGS, ifreq)
except IOError as e:
writemsg("Unable to enable loopback interface: %s\n" % (
errno.errorcode.get(e.errno, '?')),
noiselevel=-1)
sock.close()
except AttributeError:
# unshare() not supported by libc
pass
# Set requested process permissions.
if gid:
# Cast proxies to int, in case it matters.
os.setgid(int(gid))
if groups:
os.setgroups(groups)
if uid:
# Cast proxies to int, in case it matters.
os.setuid(int(uid))
if umask:
os.umask(umask)
if cwd is not None:
os.chdir(cwd)
if pre_exec:
pre_exec()
# And switch to the new process.
os.execve(binary, myargs, env)
def _spawn(self, args, fd_pipes, **kwargs):
"""
Fork a subprocess, apply local settings, and call
dblink.merge(). TODO: Share code with ForkProcess.
"""
elog_reader_fd, elog_writer_fd = os.pipe()
fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL,
fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
blockers = None
if self.blockers is not None:
# Query blockers in the main process, since closing
# of file descriptors in the subprocess can prevent
# access to open database connections such as that
# used by the sqlite metadata cache module.
blockers = self.blockers()
mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings,
treetype=self.treetype, vartree=self.vartree,
blockers=blockers, pipe=elog_writer_fd)
fd_pipes[elog_writer_fd] = elog_writer_fd
self._elog_reg_id = self.scheduler.io_add_watch(elog_reader_fd,
self._registered_events, self._elog_output_handler)
# If a concurrent emerge process tries to install a package
# in the same SLOT as this one at the same time, there is an
# extremely unlikely chance that the COUNTER values will not be
# ordered correctly unless we lock the vdb here.
# FEATURES=parallel-install skips this lock in order to
# improve performance, and the risk is practically negligible.
self._lock_vdb()
counter = None
if not self.unmerge:
counter = self.vartree.dbapi.counter_tick()
parent_pid = os.getpid()
pid = None
try:
pid = os.fork()
if pid != 0:
if not isinstance(pid, int):
raise AssertionError(
"fork returned non-integer: %s" % (repr(pid),))
os.close(elog_writer_fd)
self._elog_reader_fd = elog_reader_fd
self._buf = ""
self._elog_keys = set()
# invalidate relevant vardbapi caches
if self.vartree.dbapi._categories is not None:
self.vartree.dbapi._categories = None
self.vartree.dbapi._pkgs_changed = True
self.vartree.dbapi._clear_pkg_cache(mylink)
portage.process.spawned_pids.append(pid)
return [pid]
os.close(elog_reader_fd)
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
portage.locks._close_fds()
# We don't exec, so use close_fds=False
# (see _setup_pipes docstring).
portage.process._setup_pipes(fd_pipes, close_fds=False)
portage.output.havecolor = self.settings.get('NOCOLOR') \
not in ('yes', 'true')
# Avoid wastful updates of the vdb cache.
self.vartree.dbapi._flush_cache_enabled = False
# In this subprocess we don't want PORTAGE_BACKGROUND to
# suppress stdout/stderr output since they are pipes. We
# also don't want to open PORTAGE_LOG_FILE, since it will
# already be opened by the parent process, so we set the
# "subprocess" value for use in conditional logging code
# involving PORTAGE_LOG_FILE.
if not self.unmerge:
# unmerge phases have separate logs
if self.settings.get("PORTAGE_BACKGROUND") == "1":
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
else:
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0"
self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
self.settings["PORTAGE_BACKGROUND"] = "subprocess"
self.settings.backup_changes("PORTAGE_BACKGROUND")
rval = 1
try:
if self.unmerge:
if not mylink.exists():
rval = os.EX_OK
elif mylink.unmerge(
ldpath_mtimes=self.prev_mtimes) == os.EX_OK:
mylink.lockdb()
try:
mylink.delete()
finally:
mylink.unlockdb()
rval = os.EX_OK
else:
rval = mylink.merge(self.pkgloc, self.infloc,
myebuild=self.myebuild, mydbapi=self.mydbapi,
prev_mtimes=self.prev_mtimes, counter=counter)
except SystemExit:
raise
except:
traceback.print_exc()
finally:
os._exit(rval)
finally:
if pid == 0 or (pid is None and os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
def _exec(binary, mycommand, opt_name, fd_pipes,
env, gid, groups, uid, umask, cwd,
pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid,
unshare_flags, cgroup):
"""
Execute a given binary with options
@param binary: Name of program to execute
@type binary: String
@param mycommand: Options for program
@type mycommand: String
@param opt_name: Name of process (defaults to binary)
@type opt_name: String
@param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 }
@type fd_pipes: Dictionary
@param env: Key,Value mapping for Environmental Variables
@type env: Dictionary
@param gid: Group ID to run the process under
@type gid: Integer
@param groups: Groups the Process should be in.
@type groups: List
@param uid: User ID to run the process under
@type uid: Integer
@param umask: an int representing a unix umask (see man chmod for umask details)
@type umask: Integer
@param cwd: Current working directory
@type cwd: String
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
@param unshare_pid: If True, PID ns will be unshared from the spawned process
@type unshare_pid: Boolean
@param unshare_flags: Flags for the unshare(2) function
@type unshare_flags: Integer
@param cgroup: CGroup path to bind the process to
@type cgroup: String
@rtype: None
@return: Never returns (calls os.execve)
"""
# If the process we're creating hasn't been given a name
# assign it the name of the executable.
if not opt_name:
if binary is portage._python_interpreter:
# NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0]
# does not contain the full path of the binary.
opt_name = binary
else:
opt_name = os.path.basename(binary)
# Set up the command's argument list.
myargs = [opt_name]
myargs.extend(mycommand[1:])
# Avoid a potential UnicodeEncodeError from os.execve().
myargs = [_unicode_encode(x, encoding=_encodings['fs'],
errors='strict') for x in myargs]
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# Unregister SIGCHLD handler and wakeup_fd for the parent
# process's event loop (bug 655656).
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
try:
wakeup_fd = signal.set_wakeup_fd(-1)
if wakeup_fd > 0:
os.close(wakeup_fd)
except (ValueError, OSError):
pass
# Quiet killing of subprocesses by SIGPIPE (see bug #309001).
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
# Avoid issues triggered by inheritance of SIGQUIT handler from
# the parent process (see bug #289486).
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
_setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
# Add to cgroup
# it's better to do it from the child since we can guarantee
# it is done before we start forking children
if cgroup:
with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f:
f.write('%d\n' % os.getpid())
# Unshare (while still uid==0)
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
filename = find_library("c")
if filename is not None:
libc = LoadLibrary(filename)
if libc is not None:
try:
# Since a failed unshare call could corrupt process
# state, first validate that the call can succeed.
# The parent process should call _unshare_validate
# before it forks, so that all child processes can
# reuse _unshare_validate results that have been
# cached by the parent process.
errno_value = _unshare_validate(unshare_flags)
if errno_value == 0 and libc.unshare(unshare_flags) != 0:
errno_value = ctypes.get_errno()
if errno_value != 0:
involved_features = []
if unshare_ipc:
involved_features.append('ipc-sandbox')
if unshare_mount:
involved_features.append('mount-sandbox')
if unshare_net:
involved_features.append('network-sandbox')
if unshare_pid:
involved_features.append('pid-sandbox')
writemsg("Unable to unshare: %s (for FEATURES=\"%s\")\n" % (
errno.errorcode.get(errno_value, '?'), ' '.join(involved_features)),
noiselevel=-1)
else:
if unshare_pid:
main_child_pid = os.fork()
if main_child_pid == 0:
# pid namespace requires us to become init
binary, myargs = portage._python_interpreter, [
portage._python_interpreter,
os.path.join(portage._bin_path,
'pid-ns-init'),
_unicode_encode('' if uid is None else str(uid)),
_unicode_encode('' if gid is None else str(gid)),
_unicode_encode('' if groups is None else ','.join(str(group) for group in groups)),
_unicode_encode('' if umask is None else str(umask)),
_unicode_encode(','.join(str(fd) for fd in fd_pipes)),
binary] + myargs
uid = None
gid = None
groups = None
umask = None
else:
# Execute a supervisor process which will forward
# signals to init and forward exit status to the
# parent process. The supervisor process runs in
# the global pid namespace, so skip /proc remount
# and other setup that's intended only for the
# init process.
binary, myargs = portage._python_interpreter, [
portage._python_interpreter,
os.path.join(portage._bin_path,
'pid-ns-init'), str(main_child_pid)]
os.execve(binary, myargs, env)
if unshare_mount:
# mark the whole filesystem as slave to avoid
# mounts escaping the namespace
s = subprocess.Popen(['mount',
'--make-rslave', '/'])
mount_ret = s.wait()
if mount_ret != 0:
# TODO: should it be fatal maybe?
writemsg("Unable to mark mounts slave: %d\n" % (mount_ret,),
noiselevel=-1)
if unshare_pid:
# we need at least /proc being slave
s = subprocess.Popen(['mount',
'--make-slave', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
# can't proceed with shared /proc
writemsg("Unable to mark /proc slave: %d\n" % (mount_ret,),
noiselevel=-1)
os._exit(1)
# mount new /proc for our namespace
s = subprocess.Popen(['mount',
'-n', '-t', 'proc', 'proc', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
writemsg("Unable to mount new /proc: %d\n" % (mount_ret,),
noiselevel=-1)
os._exit(1)
if unshare_net:
# use 'localhost' to avoid hostname resolution problems
try:
socket.sethostname('localhost')
except Exception as e:
writemsg("Unable to set hostname: %s (for FEATURES=\"network-sandbox\")\n" % (
e,),
noiselevel=-1)
_configure_loopback_interface()
except AttributeError:
# unshare() not supported by libc
pass
# Set requested process permissions.
if gid:
# Cast proxies to int, in case it matters.
os.setgid(int(gid))
if groups:
os.setgroups(groups)
if uid:
# Cast proxies to int, in case it matters.
os.setuid(int(uid))
if umask:
os.umask(umask)
if cwd is not None:
os.chdir(cwd)
if pre_exec:
pre_exec()
# And switch to the new process.
os.execve(binary, myargs, env)
def get_open_fds():
return (int(fd) for fd in os.listdir("/proc/%i/fd" % os.getpid()) \
if fd.isdigit())
def hardlock_name(path):
base, tail = os.path.split(path)
return os.path.join(
base, ".%s.hardlock-%s-%s" % (tail, os.uname()[1], os.getpid()))
def _setitem(self, cpv, values):
if "_eclasses_" in values:
values = ProtectedDict(values)
values["INHERITED"] = ' '.join(sorted(values["_eclasses_"]))
new_content = []
for k in self.auxdbkey_order:
new_content.append(values.get(k, ''))
new_content.append('\n')
for i in range(magic_line_count - len(self.auxdbkey_order)):
new_content.append('\n')
new_content = ''.join(new_content)
new_content = _unicode_encode(new_content,
_encodings['repo.content'],
errors='backslashreplace')
new_fp = os.path.join(self.location, cpv)
try:
f = open(
_unicode_encode(new_fp,
encoding=_encodings['fs'],
errors='strict'), 'rb')
except EnvironmentError:
pass
else:
try:
try:
existing_st = os.fstat(f.fileno())
existing_content = f.read()
finally:
f.close()
except EnvironmentError:
pass
else:
existing_mtime = existing_st[stat.ST_MTIME]
if values['_mtime_'] == existing_mtime and \
existing_content == new_content:
return
if self.raise_stat_collision and \
values['_mtime_'] == existing_mtime and \
len(new_content) == existing_st.st_size:
raise cache_errors.StatCollision(cpv, new_fp,
existing_mtime,
existing_st.st_size)
s = cpv.rfind("/")
fp = os.path.join(self.location, cpv[:s],
".update.%i.%s" % (os.getpid(), cpv[s + 1:]))
try:
myf = open(
_unicode_encode(fp, encoding=_encodings['fs'],
errors='strict'), 'wb')
except EnvironmentError as e:
if errno.ENOENT == e.errno:
try:
self._ensure_dirs(cpv)
myf = open(
_unicode_encode(fp,
encoding=_encodings['fs'],
errors='strict'), 'wb')
except EnvironmentError as e:
raise cache_errors.CacheCorruption(cpv, e)
else:
raise cache_errors.CacheCorruption(cpv, e)
try:
myf.write(new_content)
finally:
myf.close()
self._ensure_access(fp, mtime=values["_mtime_"])
try:
os.rename(fp, new_fp)
except EnvironmentError as e:
try:
os.unlink(fp)
except EnvironmentError:
pass
raise cache_errors.CacheCorruption(cpv, e)
def _fetch_uri(self, uri):
if self.config.options.dry_run:
# Simply report success.
logging.info("dry-run: fetch '%s' from '%s'" %
(self.distfile, uri))
self._success()
self.returncode = os.EX_OK
self.wait()
return
if self.config.options.temp_dir:
self._fetch_tmp_dir_info = 'temp-dir'
distdir = self.config.options.temp_dir
else:
self._fetch_tmp_dir_info = 'distfiles'
distdir = self.config.options.distfiles
tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % os.getpid()
variables = {
"DISTDIR": distdir,
"URI": uri,
"FILE": tmp_basename
}
self._fetch_tmp_file = os.path.join(distdir, tmp_basename)
try:
os.unlink(self._fetch_tmp_file)
except OSError:
pass
args = portage.util.shlex_split(default_fetchcommand)
args = [portage.util.varexpand(x, mydict=variables)
for x in args]
if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
not os.path.isabs(args[0]):
# Python 3.1 _execvp throws TypeError for non-absolute executable
# path passed as bytes (see https://bugs.python.org/issue8513).
fullname = portage.process.find_binary(args[0])
if fullname is None:
raise portage.exception.CommandNotFound(args[0])
args[0] = fullname
args = [_unicode_encode(x,
encoding=_encodings['fs'], errors='strict') for x in args]
null_fd = os.open(os.devnull, os.O_RDONLY)
fetcher = PopenProcess(background=self.background,
proc=subprocess.Popen(args, stdin=null_fd,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT),
scheduler=self.scheduler)
os.close(null_fd)
fetcher.pipe_reader = PipeLogger(background=self.background,
input_fd=fetcher.proc.stdout, log_file_path=self._log_path,
scheduler=self.scheduler)
self._start_task(fetcher, self._fetcher_exit)
def _spawn(self, args, fd_pipes, **kwargs):
"""
Fork a subprocess, apply local settings, and call
dblink.merge(). TODO: Share code with ForkProcess.
"""
elog_reader_fd, elog_writer_fd = os.pipe()
fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL,
fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
blockers = None
if self.blockers is not None:
# Query blockers in the main process, since closing
# of file descriptors in the subprocess can prevent
# access to open database connections such as that
# used by the sqlite metadata cache module.
blockers = self.blockers()
mylink = portage.dblink(self.mycat,
self.mypkg,
settings=self.settings,
treetype=self.treetype,
vartree=self.vartree,
blockers=blockers,
pipe=elog_writer_fd)
fd_pipes[elog_writer_fd] = elog_writer_fd
self._elog_reg_id = self.scheduler.io_add_watch(
elog_reader_fd, self._registered_events, self._elog_output_handler)
# If a concurrent emerge process tries to install a package
# in the same SLOT as this one at the same time, there is an
# extremely unlikely chance that the COUNTER values will not be
# ordered correctly unless we lock the vdb here.
# FEATURES=parallel-install skips this lock in order to
# improve performance, and the risk is practically negligible.
self._lock_vdb()
counter = None
if not self.unmerge:
counter = self.vartree.dbapi.counter_tick()
parent_pid = os.getpid()
pid = None
try:
pid = os.fork()
if pid != 0:
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" %
(repr(pid), ))
os.close(elog_writer_fd)
self._elog_reader_fd = elog_reader_fd
self._buf = ""
self._elog_keys = set()
# invalidate relevant vardbapi caches
if self.vartree.dbapi._categories is not None:
self.vartree.dbapi._categories = None
self.vartree.dbapi._pkgs_changed = True
self.vartree.dbapi._clear_pkg_cache(mylink)
portage.process.spawned_pids.append(pid)
return [pid]
os.close(elog_reader_fd)
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
portage.locks._close_fds()
# We don't exec, so use close_fds=False
# (see _setup_pipes docstring).
portage.process._setup_pipes(fd_pipes, close_fds=False)
portage.output.havecolor = self.settings.get('NOCOLOR') \
not in ('yes', 'true')
# Avoid wastful updates of the vdb cache.
self.vartree.dbapi._flush_cache_enabled = False
# In this subprocess we don't want PORTAGE_BACKGROUND to
# suppress stdout/stderr output since they are pipes. We
# also don't want to open PORTAGE_LOG_FILE, since it will
# already be opened by the parent process, so we set the
# "subprocess" value for use in conditional logging code
# involving PORTAGE_LOG_FILE.
if not self.unmerge:
# unmerge phases have separate logs
if self.settings.get("PORTAGE_BACKGROUND") == "1":
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
else:
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0"
self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
self.settings["PORTAGE_BACKGROUND"] = "subprocess"
self.settings.backup_changes("PORTAGE_BACKGROUND")
rval = 1
try:
if self.unmerge:
if not mylink.exists():
rval = os.EX_OK
elif mylink.unmerge(
ldpath_mtimes=self.prev_mtimes) == os.EX_OK:
mylink.lockdb()
try:
mylink.delete()
finally:
mylink.unlockdb()
rval = os.EX_OK
else:
rval = mylink.merge(self.pkgloc,
self.infloc,
myebuild=self.myebuild,
mydbapi=self.mydbapi,
prev_mtimes=self.prev_mtimes,
counter=counter)
except SystemExit:
raise
except:
traceback.print_exc()
finally:
os._exit(rval)
finally:
if pid == 0 or (pid is None and os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
def hardlock_name(path):
return path + ".hardlock-" + os.uname()[1] + "-" + str(os.getpid())
def hardlock_name(path): return path+".hardlock-"+os.uname()[1]+"-"+str(os.getpid())
def _setitem(self, cpv, values):
if "_eclasses_" in values:
values = ProtectedDict(values)
values["INHERITED"] = ' '.join(sorted(values["_eclasses_"]))
new_content = []
for k in self.auxdbkey_order:
new_content.append(values.get(k, ''))
new_content.append('\n')
for i in range(magic_line_count - len(self.auxdbkey_order)):
new_content.append('\n')
new_content = ''.join(new_content)
new_content = _unicode_encode(new_content,
_encodings['repo.content'], errors='backslashreplace')
new_fp = os.path.join(self.location, cpv)
try:
f = open(_unicode_encode(new_fp,
encoding=_encodings['fs'], errors='strict'), 'rb')
except EnvironmentError:
pass
else:
try:
try:
existing_st = os.fstat(f.fileno())
existing_content = f.read()
finally:
f.close()
except EnvironmentError:
pass
else:
existing_mtime = existing_st[stat.ST_MTIME]
if values['_mtime_'] == existing_mtime and \
existing_content == new_content:
return
if self.raise_stat_collision and \
values['_mtime_'] == existing_mtime and \
len(new_content) == existing_st.st_size:
raise cache_errors.StatCollision(cpv, new_fp,
existing_mtime, existing_st.st_size)
s = cpv.rfind("/")
fp = os.path.join(self.location,cpv[:s],
".update.%i.%s" % (os.getpid(), cpv[s+1:]))
try:
myf = open(_unicode_encode(fp,
encoding=_encodings['fs'], errors='strict'), 'wb')
except EnvironmentError as e:
if errno.ENOENT == e.errno:
try:
self._ensure_dirs(cpv)
myf = open(_unicode_encode(fp,
encoding=_encodings['fs'], errors='strict'), 'wb')
except EnvironmentError as e:
raise cache_errors.CacheCorruption(cpv, e)
else:
raise cache_errors.CacheCorruption(cpv, e)
try:
myf.write(new_content)
finally:
myf.close()
self._ensure_access(fp, mtime=values["_mtime_"])
try:
os.rename(fp, new_fp)
except EnvironmentError as e:
try:
os.unlink(fp)
except EnvironmentError:
pass
raise cache_errors.CacheCorruption(cpv, e)
def _fetch_uri(self, uri):
if self.config.options.dry_run:
# Simply report success.
logging.info("dry-run: fetch '%s' from '%s'" %
(self.distfile, uri))
self._success()
self.returncode = os.EX_OK
self.wait()
return
if self.config.options.temp_dir:
self._fetch_tmp_dir_info = 'temp-dir'
distdir = self.config.options.temp_dir
else:
self._fetch_tmp_dir_info = 'distfiles'
distdir = self.config.options.distfiles
tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % os.getpid()
variables = {"DISTDIR": distdir, "URI": uri, "FILE": tmp_basename}
self._fetch_tmp_file = os.path.join(distdir, tmp_basename)
try:
os.unlink(self._fetch_tmp_file)
except OSError:
pass
args = portage.util.shlex_split(default_fetchcommand)
args = [portage.util.varexpand(x, mydict=variables) for x in args]
if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \
not os.path.isabs(args[0]):
# Python 3.1 _execvp throws TypeError for non-absolute executable
# path passed as bytes (see http://bugs.python.org/issue8513).
fullname = portage.process.find_binary(args[0])
if fullname is None:
raise portage.exception.CommandNotFound(args[0])
args[0] = fullname
args = [
_unicode_encode(x, encoding=_encodings['fs'], errors='strict')
for x in args
]
null_fd = os.open(os.devnull, os.O_RDONLY)
fetcher = PopenProcess(background=self.background,
proc=subprocess.Popen(args,
stdin=null_fd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT),
scheduler=self.scheduler)
os.close(null_fd)
fetcher.pipe_reader = PipeLogger(background=self.background,
input_fd=fetcher.proc.stdout,
log_file_path=self._log_path,
scheduler=self.scheduler)
self._start_task(fetcher, self._fetcher_exit)
def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
uid=None, gid=None, groups=None, umask=None, logfile=None,
path_lookup=True, pre_exec=None, close_fds=True, unshare_net=False,
unshare_ipc=False, cgroup=None):
"""
Spawns a given command.
@param mycommand: the command to execute
@type mycommand: String or List (Popen style list)
@param env: A dict of Key=Value pairs for env variables
@type env: Dictionary
@param opt_name: an optional name for the spawn'd process (defaults to the binary name)
@type opt_name: String
@param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example
(default is {0:stdin, 1:stdout, 2:stderr})
@type fd_pipes: Dictionary
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them.
@type returnpid: Boolean
@param uid: User ID to spawn as; useful for dropping privilages
@type uid: Integer
@param gid: Group ID to spawn as; useful for dropping privilages
@type gid: Integer
@param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts.
@type groups: List
@param umask: An integer representing the umask for the process (see man chmod for umask details)
@type umask: Integer
@param logfile: name of a file to use for logging purposes
@type logfile: String
@param path_lookup: If the binary is not fully specified then look for it in PATH
@type path_lookup: Boolean
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param close_fds: If True, then close all file descriptors except those
referenced by fd_pipes (default is True).
@type close_fds: Boolean
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
logfile requires stdout and stderr to be assigned to this process (ie not pointed
somewhere else.)
"""
# mycommand is either a str or a list
if isinstance(mycommand, basestring):
mycommand = mycommand.split()
if sys.hexversion < 0x3000000:
# Avoid a potential UnicodeEncodeError from os.execve().
env_bytes = {}
for k, v in env.items():
env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \
_unicode_encode(v, encoding=_encodings['content'])
env = env_bytes
del env_bytes
# If an absolute path to an executable file isn't given
# search for it unless we've been told not to.
binary = mycommand[0]
if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \
(not os.path.isabs(binary) or not os.path.isfile(binary)
or not os.access(binary, os.X_OK)):
binary = path_lookup and find_binary(binary) or None
if not binary:
raise CommandNotFound(mycommand[0])
# If we haven't been told what file descriptors to use
# default to propagating our stdin, stdout and stderr.
if fd_pipes is None:
fd_pipes = {
0:portage._get_stdin().fileno(),
1:sys.__stdout__.fileno(),
2:sys.__stderr__.fileno(),
}
# mypids will hold the pids of all processes created.
mypids = []
if logfile:
# Using a log file requires that stdout and stderr
# are assigned to the process we're running.
if 1 not in fd_pipes or 2 not in fd_pipes:
raise ValueError(fd_pipes)
# Create a pipe
(pr, pw) = os.pipe()
# Create a tee process, giving it our stdout and stderr
# as well as the read end of the pipe.
mypids.extend(spawn(('tee', '-i', '-a', logfile),
returnpid=True, fd_pipes={0:pr,
1:fd_pipes[1], 2:fd_pipes[2]}))
# We don't need the read end of the pipe, so close it.
os.close(pr)
# Assign the write end of the pipe to our stdout and stderr.
fd_pipes[1] = pw
fd_pipes[2] = pw
# This caches the libc library lookup in the current
# process, so that it's only done once rather than
# for each child process.
if unshare_net or unshare_ipc:
find_library("c")
parent_pid = os.getpid()
pid = None
try:
pid = os.fork()
if pid == 0:
try:
_exec(binary, mycommand, opt_name, fd_pipes,
env, gid, groups, uid, umask, pre_exec, close_fds,
unshare_net, unshare_ipc, cgroup)
except SystemExit:
raise
except Exception as e:
# We need to catch _any_ exception so that it doesn't
# propagate out of this function and cause exiting
# with anything other than os._exit()
writemsg("%s:\n %s\n" % (e, " ".join(mycommand)),
noiselevel=-1)
traceback.print_exc()
sys.stderr.flush()
finally:
if pid == 0 or (pid is None and os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" % (repr(pid),))
# Add the pid to our local and the global pid lists.
mypids.append(pid)
# If we started a tee process the write side of the pipe is no
# longer needed, so close it.
if logfile:
os.close(pw)
# If the caller wants to handle cleaning up the processes, we tell
# it about all processes that were created.
if returnpid:
return mypids
# Otherwise we clean them up.
while mypids:
# Pull the last reader in the pipe chain. If all processes
# in the pipe are well behaved, it will die when the process
# it is reading from dies.
pid = mypids.pop(0)
# and wait for it.
retval = os.waitpid(pid, 0)[1]
if retval:
# If it failed, kill off anything else that
# isn't dead yet.
for pid in mypids:
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
if os.waitpid(pid, os.WNOHANG)[0] == 0:
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0)
# If it got a signal, return the signal that was sent.
if (retval & 0xff):
return ((retval & 0xff) << 8)
# Otherwise, return its exit code.
return (retval >> 8)
# Everything succeeded
return 0
