def _ensure_dirs(self, path=None): """with path!=None, ensure beyond self.location. otherwise, ensure self.location""" if path: path = os.path.dirname(path) base = self.location else: path = self.location base='/' for dir in path.lstrip(os.path.sep).rstrip(os.path.sep).split(os.path.sep): base = os.path.join(base,dir) if not os.path.exists(base): if self._perms != -1: um = os.umask(0) try: perms = self._perms if perms == -1: perms = 0 perms |= 0o755 os.mkdir(base, perms) if self._gid != -1: os.chown(base, -1, self._gid) finally: if self._perms != -1: os.umask(um)
def read_config(mandatory_opts):
eprefix = portage.settings["EPREFIX"]
if portage._not_installed:
config_path = os.path.join(portage.PORTAGE_BASE_PATH, "cnf",
"dispatch-conf.conf")
else:
config_path = os.path.join(eprefix or os.sep, "etc/dispatch-conf.conf")
loader = KeyValuePairFileLoader(config_path, None)
opts, _errors = loader.load()
if not opts:
print(
_("dispatch-conf: Error reading {}; fatal").format(config_path),
file=sys.stderr,
)
sys.exit(1)
# Handle quote removal here, since KeyValuePairFileLoader doesn't do that.
quotes = "\"'"
for k, v in opts.items():
if v[:1] in quotes and v[:1] == v[-1:]:
opts[k] = v[1:-1]
for key in mandatory_opts:
if key not in opts:
if key == "merge":
opts[
"merge"] = "sdiff --suppress-common-lines --output='%s' '%s' '%s'"
else:
print(
_('dispatch-conf: Missing option "%s" in /etc/dispatch-conf.conf; fatal'
) % (key, ),
file=sys.stderr,
)
# archive-dir supports ${EPREFIX} expansion, in order to avoid hardcoding
variables = {"EPREFIX": eprefix}
opts["archive-dir"] = varexpand(opts["archive-dir"], mydict=variables)
if not os.path.exists(opts["archive-dir"]):
os.mkdir(opts["archive-dir"])
# Use restrictive permissions by default, in order to protect
# against vulnerabilities (like bug #315603 involving rcs).
os.chmod(opts["archive-dir"], 0o700)
elif not os.path.isdir(opts["archive-dir"]):
print(
_("dispatch-conf: Config archive dir [%s] must exist; fatal") %
(opts["archive-dir"], ),
file=sys.stderr,
)
sys.exit(1)
return opts
def read_config(mandatory_opts):
eprefix = portage.settings["EPREFIX"]
if portage._not_installed:
config_path = os.path.join(portage.PORTAGE_BASE_PATH, "cnf", "dispatch-conf.conf")
else:
config_path = os.path.join(eprefix or os.sep, "etc/dispatch-conf.conf")
loader = KeyValuePairFileLoader(config_path, None)
opts, _errors = loader.load()
if not opts:
print(_('dispatch-conf: Error reading /etc/dispatch-conf.conf; fatal'), file=sys.stderr)
sys.exit(1)
# Handle quote removal here, since KeyValuePairFileLoader doesn't do that.
quotes = "\"'"
for k, v in opts.items():
if v[:1] in quotes and v[:1] == v[-1:]:
opts[k] = v[1:-1]
for key in mandatory_opts:
if key not in opts:
if key == "merge":
opts["merge"] = "sdiff --suppress-common-lines --output='%s' '%s' '%s'"
else:
print(_('dispatch-conf: Missing option "%s" in /etc/dispatch-conf.conf; fatal') % (key,), file=sys.stderr)
# archive-dir supports ${EPREFIX} expansion, in order to avoid hardcoding
variables = {"EPREFIX": eprefix}
opts['archive-dir'] = varexpand(opts['archive-dir'], mydict=variables)
if not os.path.exists(opts['archive-dir']):
os.mkdir(opts['archive-dir'])
# Use restrictive permissions by default, in order to protect
# against vulnerabilities (like bug #315603 involving rcs).
os.chmod(opts['archive-dir'], 0o700)
elif not os.path.isdir(opts['archive-dir']):
print(_('dispatch-conf: Config archive dir [%s] must exist; fatal') % (opts['archive-dir'],), file=sys.stderr)
sys.exit(1)
return opts
def testSymlinkDir(self):
"""
Test that masked symlinks to directories are removed.
"""
tmp_dir = tempfile.mkdtemp()
try:
base_dir = os.path.join(tmp_dir, "foo")
target_dir = os.path.join(tmp_dir, "foo", "bar")
link_name = os.path.join(tmp_dir, "foo", "baz")
os.mkdir(base_dir)
os.mkdir(target_dir)
os.symlink(target_dir, link_name)
install_mask = InstallMask("/foo/")
install_mask_dir(tmp_dir, install_mask)
self.assertFalse(os.path.lexists(link_name),
"failed to remove {}".format(link_name))
self.assertFalse(os.path.lexists(base_dir),
"failed to remove {}".format(base_dir))
finally:
shutil.rmtree(tmp_dir)
def binTestsInit(): binTestsCleanup() global basedir, env basedir = tempfile.mkdtemp() env = os.environ.copy() env["D"] = os.path.join(basedir, "image") env["T"] = os.path.join(basedir, "temp") env["S"] = os.path.join(basedir, "workdir") env["PF"] = "portage-tests-0.09-r1" env["PATH"] = bindir + ":" + env["PATH"] env["PORTAGE_BIN_PATH"] = bindir env["PORTAGE_PYM_PATH"] = pymdir os.mkdir(env["D"]) os.mkdir(env["T"]) os.mkdir(env["S"])
def binTestsInit():
binTestsCleanup()
global basedir, env
basedir = tempfile.mkdtemp()
env = os.environ.copy()
env["D"] = os.path.join(basedir, "image")
env["T"] = os.path.join(basedir, "temp")
env["S"] = os.path.join(basedir, "workdir")
env["PF"] = "portage-tests-0.09-r1"
env["PATH"] = bindir + ":" + env["PATH"]
env["PORTAGE_BIN_PATH"] = bindir
env["PORTAGE_PYM_PATH"] = pymdir
os.mkdir(env["D"])
os.mkdir(env["T"])
os.mkdir(env["S"])
def binTestsInit():
binTestsCleanup()
global basedir, env
basedir = tempfile.mkdtemp()
env = {}
env['EAPI'] = '0'
env['D'] = os.path.join(basedir, 'image')
env['T'] = os.path.join(basedir, 'temp')
env['S'] = os.path.join(basedir, 'workdir')
env['PF'] = 'portage-tests-0.09-r1'
env['PATH'] = bindir + ':' + os.environ['PATH']
env['PORTAGE_BIN_PATH'] = bindir
env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
env['PORTAGE_INST_UID'] = str(os.getuid())
env['PORTAGE_INST_GID'] = str(os.getgid())
env['DESTTREE'] = '/usr'
os.mkdir(env['D'])
os.mkdir(env['T'])
os.mkdir(env['S'])
def binTestsInit():
binTestsCleanup()
global basedir, env
basedir = tempfile.mkdtemp()
env = {}
env["EAPI"] = "0"
env["D"] = os.path.join(basedir, "image")
env["T"] = os.path.join(basedir, "temp")
env["S"] = os.path.join(basedir, "workdir")
env["PF"] = "portage-tests-0.09-r1"
env["PATH"] = bindir + ":" + os.environ["PATH"]
env["PORTAGE_BIN_PATH"] = bindir
env["PORTAGE_PYM_PATH"] = pymdir
env["PORTAGE_INST_UID"] = str(os.getuid())
env["PORTAGE_INST_GID"] = str(os.getgid())
env["DESTTREE"] = "/usr"
os.mkdir(env["D"])
os.mkdir(env["T"])
os.mkdir(env["S"])
def binTestsInit():
binTestsCleanup()
global basedir, env
basedir = tempfile.mkdtemp()
env = {}
env['EAPI'] = '0'
env['D'] = os.path.join(basedir, 'image')
env['T'] = os.path.join(basedir, 'temp')
env['S'] = os.path.join(basedir, 'workdir')
env['PF'] = 'portage-tests-0.09-r1'
env['PATH'] = bindir + ':' + os.environ['PATH']
env['PORTAGE_BIN_PATH'] = bindir
env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
env['PORTAGE_INST_UID'] = str(os.getuid())
env['PORTAGE_INST_GID'] = str(os.getgid())
env['DESTTREE'] = '/usr'
os.mkdir(env['D'])
os.mkdir(env['T'])
os.mkdir(env['S'])
def binTestsInit(): binTestsCleanup() global basedir, env basedir = tempfile.mkdtemp() env = os.environ.copy() env["EAPI"] = "0" env["D"] = os.path.join(basedir, "image") env["T"] = os.path.join(basedir, "temp") env["S"] = os.path.join(basedir, "workdir") env["PF"] = "portage-tests-0.09-r1" env["PATH"] = bindir + ":" + env["PATH"] env["PORTAGE_BIN_PATH"] = bindir env["PORTAGE_PYM_PATH"] = pymdir env["PORTAGE_INST_UID"] = str(os.getuid()) env["PORTAGE_INST_GID"] = str(os.getgid()) env["DESTTREE"] = "/usr" os.mkdir(env["D"]) os.mkdir(env["T"]) os.mkdir(env["S"])
def testSyncLocal(self):
debug = False
skip_reason = self._must_skip()
if skip_reason:
self.portage_skip = skip_reason
self.assertFalse(True, skip_reason)
return
repos_conf = textwrap.dedent("""
[DEFAULT]
%(default_keys)s
[test_repo]
location = %(EPREFIX)s/var/repositories/test_repo
sync-type = %(sync-type)s
sync-depth = %(sync-depth)s
sync-uri = file://%(EPREFIX)s/var/repositories/test_repo_sync
sync-rcu = %(sync-rcu)s
sync-rcu-store-dir = %(EPREFIX)s/var/repositories/test_repo_rcu_storedir
auto-sync = %(auto-sync)s
%(repo_extra_keys)s
""")
profile = {
"eapi": ("5", ),
"package.use.stable.mask": ("dev-libs/A flag", )
}
ebuilds = {"dev-libs/A-0": {}}
user_config = {'make.conf': ('FEATURES="metadata-transfer"', )}
playground = ResolverPlayground(ebuilds=ebuilds,
profile=profile,
user_config=user_config,
debug=debug)
settings = playground.settings
eprefix = settings["EPREFIX"]
eroot = settings["EROOT"]
homedir = os.path.join(eroot, "home")
distdir = os.path.join(eprefix, "distdir")
repo = settings.repositories["test_repo"]
metadata_dir = os.path.join(repo.location, "metadata")
cmds = {}
for cmd in ("emerge", "emaint"):
for bindir in (self.bindir, self.sbindir):
path = os.path.join(bindir, cmd)
if os.path.exists(path):
cmds[cmd] = (portage._python_interpreter, "-b", "-Wd",
path)
break
else:
raise AssertionError('%s binary not found in %s or %s' %
(cmd, self.bindir, self.sbindir))
git_binary = find_binary("git")
git_cmd = (git_binary, )
committer_name = "Gentoo Dev"
committer_email = "*****@*****.**"
def repos_set_conf(sync_type,
dflt_keys=None,
xtra_keys=None,
auto_sync="yes",
sync_rcu=False,
sync_depth=None):
env["PORTAGE_REPOSITORIES"] = repos_conf % {\
"EPREFIX": eprefix, "sync-type": sync_type,
"sync-depth": 0 if sync_depth is None else sync_depth,
"sync-rcu": "yes" if sync_rcu else "no",
"auto-sync": auto_sync,
"default_keys": "" if dflt_keys is None else dflt_keys,
"repo_extra_keys": "" if xtra_keys is None else xtra_keys}
def alter_ebuild():
with open(
os.path.join(repo.location + "_sync", "dev-libs", "A",
"A-0.ebuild"), "a") as f:
f.write("\n")
bump_timestamp()
def bump_timestamp():
bump_timestamp.timestamp += datetime.timedelta(seconds=1)
with open(
os.path.join(repo.location + '_sync', 'metadata',
'timestamp.chk'), 'w') as f:
f.write(
bump_timestamp.timestamp.strftime(
'%s\n' % TIMESTAMP_FORMAT, ))
bump_timestamp.timestamp = datetime.datetime.utcnow()
bump_timestamp_cmds = ((homedir, bump_timestamp), )
sync_cmds = (
(homedir, cmds["emerge"] + ("--sync", )),
(homedir, lambda: self.assertTrue(
os.path.exists(os.path.join(repo.location, "dev-libs", "A")),
"dev-libs/A expected, but missing")),
(homedir, cmds["emaint"] + ("sync", "-A")),
)
sync_cmds_auto_sync = (
(homedir, lambda: repos_set_conf("rsync", auto_sync="no")),
(homedir, cmds["emerge"] + ("--sync", )),
(homedir, lambda: self.assertFalse(
os.path.exists(os.path.join(repo.location, "dev-libs", "A")),
"dev-libs/A found, expected missing")),
(homedir, lambda: repos_set_conf("rsync", auto_sync="yes")),
)
rename_repo = (
(homedir,
lambda: os.rename(repo.location, repo.location + "_sync")), )
rsync_opts_repos = (
(homedir, alter_ebuild),
(homedir, lambda: repos_set_conf(
"rsync", None,
"sync-rsync-extra-opts = --backup --backup-dir=%s" %
_shell_quote(repo.location + "_back"))),
(homedir, cmds['emerge'] + ("--sync", )),
(homedir,
lambda: self.assertTrue(os.path.exists(repo.location + "_back"))),
(homedir, lambda: shutil.rmtree(repo.location + "_back")),
(homedir, lambda: repos_set_conf("rsync")),
)
rsync_opts_repos_default = (
(homedir, alter_ebuild),
(homedir, lambda: repos_set_conf(
"rsync", "sync-rsync-extra-opts = --backup --backup-dir=%s" %
_shell_quote(repo.location + "_back"))),
(homedir, cmds['emerge'] + ("--sync", )),
(homedir,
lambda: self.assertTrue(os.path.exists(repo.location + "_back"))),
(homedir, lambda: shutil.rmtree(repo.location + "_back")),
(homedir, lambda: repos_set_conf("rsync")),
)
rsync_opts_repos_default_ovr = (
(homedir, alter_ebuild),
(homedir, lambda: repos_set_conf(
"rsync", "sync-rsync-extra-opts = --backup --backup-dir=%s" %
_shell_quote(repo.location + "_back_nowhere"),
"sync-rsync-extra-opts = --backup --backup-dir=%s" %
_shell_quote(repo.location + "_back"))),
(homedir, cmds['emerge'] + ("--sync", )),
(homedir,
lambda: self.assertTrue(os.path.exists(repo.location + "_back"))),
(homedir, lambda: shutil.rmtree(repo.location + "_back")),
(homedir, lambda: repos_set_conf("rsync")),
)
rsync_opts_repos_default_cancel = (
(homedir, alter_ebuild),
(homedir, lambda: repos_set_conf(
"rsync", "sync-rsync-extra-opts = --backup --backup-dir=%s" %
_shell_quote(repo.location + "_back_nowhere"
), "sync-rsync-extra-opts = ")),
(homedir, cmds['emerge'] + ("--sync", )),
(homedir,
lambda: self.assertFalse(os.path.exists(repo.location + "_back"))
),
(homedir, lambda: repos_set_conf("rsync")),
)
delete_repo_location = (
(homedir, lambda: shutil.rmtree(repo.user_location)),
(homedir, lambda: os.mkdir(repo.user_location)),
)
revert_rcu_layout = (
(homedir,
lambda: os.rename(repo.user_location, repo.user_location + '.bak')
),
(homedir,
lambda: os.rename(os.path.realpath(repo.user_location + '.bak'),
repo.user_location)),
(homedir, lambda: os.unlink(repo.user_location + '.bak')),
(homedir,
lambda: shutil.rmtree(repo.user_location + '_rcu_storedir')),
)
upstream_git_commit = (
(
repo.location + "_sync",
git_cmd +
('commit', '--allow-empty', '-m', 'test empty commit'),
),
(
repo.location + "_sync",
git_cmd +
('commit', '--allow-empty', '-m', 'test empty commit 2'),
),
)
delete_sync_repo = ((homedir,
lambda: shutil.rmtree(repo.location + "_sync")), )
git_repo_create = (
(repo.location, git_cmd + (
"config",
"--global",
"user.name",
committer_name,
)),
(repo.location, git_cmd + (
"config",
"--global",
"user.email",
committer_email,
)),
(repo.location, git_cmd + ("init-db", )),
(repo.location, git_cmd + ("add", ".")),
(repo.location,
git_cmd + ("commit", "-a", "-m", "add whole repo")),
)
sync_type_git = ((homedir, lambda: repos_set_conf("git")), )
sync_type_git_shallow = ((
homedir, lambda: repos_set_conf("git", sync_depth=1)), )
sync_rsync_rcu = ((homedir,
lambda: repos_set_conf("rsync", sync_rcu=True)), )
pythonpath = os.environ.get("PYTHONPATH")
if pythonpath is not None and not pythonpath.strip():
pythonpath = None
if pythonpath is not None and \
pythonpath.split(":")[0] == PORTAGE_PYM_PATH:
pass
else:
if pythonpath is None:
pythonpath = ""
else:
pythonpath = ":" + pythonpath
pythonpath = PORTAGE_PYM_PATH + pythonpath
env = {
"PORTAGE_OVERRIDE_EPREFIX":
eprefix,
"DISTDIR":
distdir,
"GENTOO_COMMITTER_NAME":
committer_name,
"GENTOO_COMMITTER_EMAIL":
committer_email,
"HOME":
homedir,
"PATH":
os.environ["PATH"],
"PORTAGE_GRPNAME":
os.environ["PORTAGE_GRPNAME"],
"PORTAGE_USERNAME":
os.environ["PORTAGE_USERNAME"],
"PYTHONDONTWRITEBYTECODE":
os.environ.get("PYTHONDONTWRITEBYTECODE", ""),
"PYTHONPATH":
pythonpath,
}
repos_set_conf("rsync")
if os.environ.get("SANDBOX_ON") == "1":
# avoid problems from nested sandbox instances
env["FEATURES"] = "-sandbox -usersandbox"
dirs = [homedir, metadata_dir]
try:
for d in dirs:
ensure_dirs(d)
timestamp_path = os.path.join(metadata_dir, 'timestamp.chk')
with open(timestamp_path, 'w') as f:
f.write(
bump_timestamp.timestamp.strftime(
'%s\n' % TIMESTAMP_FORMAT, ))
if debug:
# The subprocess inherits both stdout and stderr, for
# debugging purposes.
stdout = None
else:
# The subprocess inherits stderr so that any warnings
# triggered by python -Wd will be visible.
stdout = subprocess.PIPE
for cwd, cmd in rename_repo + sync_cmds_auto_sync + sync_cmds + \
rsync_opts_repos + rsync_opts_repos_default + \
rsync_opts_repos_default_ovr + rsync_opts_repos_default_cancel + \
bump_timestamp_cmds + sync_rsync_rcu + sync_cmds + revert_rcu_layout + \
delete_repo_location + sync_cmds + sync_cmds + \
bump_timestamp_cmds + sync_cmds + revert_rcu_layout + \
delete_sync_repo + git_repo_create + sync_type_git + \
rename_repo + sync_cmds + upstream_git_commit + sync_cmds + \
sync_type_git_shallow + upstream_git_commit + sync_cmds:
if hasattr(cmd, '__call__'):
cmd()
continue
abs_cwd = os.path.join(repo.location, cwd)
proc = subprocess.Popen(cmd,
cwd=abs_cwd,
env=env,
stdout=stdout)
if debug:
proc.wait()
else:
output = proc.stdout.readlines()
proc.wait()
proc.stdout.close()
if proc.returncode != os.EX_OK:
for line in output:
sys.stderr.write(_unicode_decode(line))
self.assertEqual(os.EX_OK, proc.returncode,
"%s failed in %s" % (
cmd,
cwd,
))
finally:
playground.cleanup()
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and not os.path.isdir(
self.settings["PORTAGE_BUILDDIR"]):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % (
self.phase, self.settings["PORTAGE_BUILDDIR"])
self._eerror(textwrap.wrap(msg, 72))
self.returncode = 1
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == "Linux"
and "cgroup" in self.settings.features
and self.phase not in _global_pid_phases):
cgroup_root = "/sys/fs/cgroup"
cgroup_portage = os.path.join(cgroup_root, "portage")
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call([
"mount",
"-t",
"tmpfs",
"-o",
"rw,nosuid,nodev,noexec,mode=0755",
"tmpfs",
cgroup_root,
])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call([
"mount",
"-t",
"cgroup",
"-o",
"rw,nosuid,nodev,noexec,none,name=portage",
"tmpfs",
cgroup_portage,
])
with open(os.path.join(cgroup_portage, "release_agent"),
"w") as f:
f.write(
os.path.join(
self.settings["PORTAGE_BIN_PATH"],
"cgroup-release-agent",
))
with open(
os.path.join(cgroup_portage, "notify_on_release"),
"w") as f:
f.write("1")
else:
# Update release_agent if it no longer exists, because
# it refers to a temporary path when portage is updating
# itself.
release_agent = os.path.join(cgroup_portage,
"release_agent")
try:
with open(release_agent) as f:
release_agent_path = f.readline().rstrip("\n")
except EnvironmentError:
release_agent_path = None
if release_agent_path is None or not os.path.exists(
release_agent_path):
with open(release_agent, "w") as f:
f.write(
os.path.join(
self.settings["PORTAGE_BIN_PATH"],
"cgroup-release-agent",
))
cgroup_path = tempfile.mkdtemp(
dir=cgroup_portage,
prefix="%s:%s." %
(self.settings["CATEGORY"], self.settings["PF"]),
)
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings["NOCOLOR"] = "true"
start_ipc_daemon = False
if self._enable_ipc_daemon:
self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None)
if self.phase not in self._phases_without_builddir:
start_ipc_daemon = True
if "PORTAGE_BUILDDIR_LOCKED" not in self.settings:
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._start_future = self._build_dir.async_lock()
self._start_future.add_done_callback(
functools.partial(
self._start_post_builddir_lock,
start_ipc_daemon=start_ipc_daemon,
))
return
else:
self.settings.pop("PORTAGE_IPC_DAEMON", None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop("PORTAGE_IPC_DAEMON", None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(self.settings["PORTAGE_BUILDDIR"],
".exit_status")
self.settings["PORTAGE_EBUILD_EXIT_FILE"] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None)
self._start_post_builddir_lock(start_ipc_daemon=start_ipc_daemon)
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and \
not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % \
(self.phase, self.settings['PORTAGE_BUILDDIR'])
self._eerror(textwrap.wrap(msg, 72))
self._set_returncode((self.pid, 1 << 8))
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == 'Linux'
and 'cgroup' in self.settings.features
and self.phase not in self._phases_without_cgroup):
cgroup_root = '/sys/fs/cgroup'
cgroup_portage = os.path.join(cgroup_root, 'portage')
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call(['mount', '-t', 'tmpfs',
'-o', 'rw,nosuid,nodev,noexec,mode=0755',
'tmpfs', cgroup_root])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call(['mount', '-t', 'cgroup',
'-o', 'rw,nosuid,nodev,noexec,none,name=portage',
'tmpfs', cgroup_portage])
cgroup_path = tempfile.mkdtemp(dir=cgroup_portage,
prefix='%s:%s.' % (self.settings["CATEGORY"],
self.settings["PF"]))
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings:
self._build_dir = EbuildBuildDir(
scheduler=self.scheduler, settings=self.settings)
self._build_dir.lock()
self.settings['PORTAGE_IPC_DAEMON'] = "1"
self._start_ipc_daemon()
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(
self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.fd_pipes is None:
self.fd_pipes = {}
null_fd = None
if 0 not in self.fd_pipes and \
self.phase not in self._phases_interactive_whitelist and \
"interactive" not in self.settings.get("PROPERTIES", "").split():
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
try:
SpawnProcess._start(self)
finally:
if null_fd is not None:
os.close(null_fd)
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and \
not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % \
(self.phase, self.settings['PORTAGE_BUILDDIR'])
self._eerror(textwrap.wrap(msg, 72))
self._set_returncode((self.pid, 1 << 8))
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == 'Linux'
and 'cgroup' in self.settings.features
and self.phase not in self._phases_without_cgroup):
cgroup_root = '/sys/fs/cgroup'
cgroup_portage = os.path.join(cgroup_root, 'portage')
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call([
'mount', '-t', 'tmpfs', '-o',
'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs',
cgroup_root
])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call([
'mount', '-t', 'cgroup', '-o',
'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs',
cgroup_portage
])
cgroup_path = tempfile.mkdtemp(
dir=cgroup_portage,
prefix='%s:%s.' %
(self.settings["CATEGORY"], self.settings["PF"]))
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings:
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._build_dir.lock()
self.settings['PORTAGE_IPC_DAEMON'] = "1"
self._start_ipc_daemon()
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.fd_pipes is None:
self.fd_pipes = {}
null_fd = None
if 0 not in self.fd_pipes and \
self.phase not in self._phases_interactive_whitelist and \
"interactive" not in self.settings.get("PROPERTIES", "").split():
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
try:
SpawnProcess._start(self)
finally:
if null_fd is not None:
os.close(null_fd)
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and \
not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % \
(self.phase, self.settings['PORTAGE_BUILDDIR'])
self._eerror(textwrap.wrap(msg, 72))
self.returncode = 1
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == 'Linux'
and 'cgroup' in self.settings.features
and self.phase not in _global_pid_phases):
cgroup_root = '/sys/fs/cgroup'
cgroup_portage = os.path.join(cgroup_root, 'portage')
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call(['mount', '-t', 'tmpfs',
'-o', 'rw,nosuid,nodev,noexec,mode=0755',
'tmpfs', cgroup_root])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call(['mount', '-t', 'cgroup',
'-o', 'rw,nosuid,nodev,noexec,none,name=portage',
'tmpfs', cgroup_portage])
with open(os.path.join(
cgroup_portage, 'release_agent'), 'w') as f:
f.write(os.path.join(self.settings['PORTAGE_BIN_PATH'],
'cgroup-release-agent'))
with open(os.path.join(
cgroup_portage, 'notify_on_release'), 'w') as f:
f.write('1')
else:
# Update release_agent if it no longer exists, because
# it refers to a temporary path when portage is updating
# itself.
release_agent = os.path.join(
cgroup_portage, 'release_agent')
try:
with open(release_agent) as f:
release_agent_path = f.readline().rstrip('\n')
except EnvironmentError:
release_agent_path = None
if (release_agent_path is None or
not os.path.exists(release_agent_path)):
with open(release_agent, 'w') as f:
f.write(os.path.join(
self.settings['PORTAGE_BIN_PATH'],
'cgroup-release-agent'))
cgroup_path = tempfile.mkdtemp(dir=cgroup_portage,
prefix='%s:%s.' % (self.settings["CATEGORY"],
self.settings["PF"]))
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
start_ipc_daemon = False
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
start_ipc_daemon = True
if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings:
self._build_dir = EbuildBuildDir(
scheduler=self.scheduler, settings=self.settings)
self._start_future = self._build_dir.async_lock()
self._start_future.add_done_callback(
functools.partial(self._start_post_builddir_lock,
start_ipc_daemon=start_ipc_daemon))
return
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(
self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
self._start_post_builddir_lock(start_ipc_daemon=start_ipc_daemon)
def testSyncLocal(self):
debug = False
skip_reason = self._must_skip()
if skip_reason:
self.portage_skip = skip_reason
self.assertFalse(True, skip_reason)
return
repos_conf = textwrap.dedent("""
[DEFAULT]
%(default_keys)s
[test_repo]
location = %(EPREFIX)s/var/repositories/test_repo
sync-type = %(sync-type)s
sync-depth = %(sync-depth)s
sync-uri = file://%(EPREFIX)s/var/repositories/test_repo_sync
sync-rcu = %(sync-rcu)s
sync-rcu-store-dir = %(EPREFIX)s/var/repositories/test_repo_rcu_storedir
auto-sync = %(auto-sync)s
%(repo_extra_keys)s
""")
profile = {
"eapi": ("5",),
"package.use.stable.mask": ("dev-libs/A flag",)
}
ebuilds = {
"dev-libs/A-0": {}
}
user_config = {
'make.conf': ('FEATURES="metadata-transfer"',)
}
playground = ResolverPlayground(ebuilds=ebuilds,
profile=profile, user_config=user_config, debug=debug)
settings = playground.settings
eprefix = settings["EPREFIX"]
eroot = settings["EROOT"]
homedir = os.path.join(eroot, "home")
distdir = os.path.join(eprefix, "distdir")
repo = settings.repositories["test_repo"]
metadata_dir = os.path.join(repo.location, "metadata")
cmds = {}
for cmd in ("emerge", "emaint"):
for bindir in (self.bindir, self.sbindir):
path = os.path.join(bindir, cmd)
if os.path.exists(path):
cmds[cmd] = (portage._python_interpreter,
"-b", "-Wd", path)
break
else:
raise AssertionError('%s binary not found in %s or %s' %
(cmd, self.bindir, self.sbindir))
git_binary = find_binary("git")
git_cmd = (git_binary,)
committer_name = "Gentoo Dev"
committer_email = "*****@*****.**"
def repos_set_conf(sync_type, dflt_keys=None, xtra_keys=None,
auto_sync="yes", sync_rcu=False, sync_depth=None):
env["PORTAGE_REPOSITORIES"] = repos_conf % {\
"EPREFIX": eprefix, "sync-type": sync_type,
"sync-depth": 0 if sync_depth is None else sync_depth,
"sync-rcu": "yes" if sync_rcu else "no",
"auto-sync": auto_sync,
"default_keys": "" if dflt_keys is None else dflt_keys,
"repo_extra_keys": "" if xtra_keys is None else xtra_keys}
def alter_ebuild():
with open(os.path.join(repo.location + "_sync",
"dev-libs", "A", "A-0.ebuild"), "a") as f:
f.write("\n")
bump_timestamp()
def bump_timestamp():
bump_timestamp.timestamp += datetime.timedelta(seconds=1)
with open(os.path.join(repo.location + '_sync', 'metadata', 'timestamp.chk'), 'w') as f:
f.write(bump_timestamp.timestamp.strftime('%s\n' % TIMESTAMP_FORMAT,))
bump_timestamp.timestamp = datetime.datetime.utcnow()
bump_timestamp_cmds = (
(homedir, bump_timestamp),
)
sync_cmds = (
(homedir, cmds["emerge"] + ("--sync",)),
(homedir, lambda: self.assertTrue(os.path.exists(
os.path.join(repo.location, "dev-libs", "A")
), "dev-libs/A expected, but missing")),
(homedir, cmds["emaint"] + ("sync", "-A")),
)
sync_cmds_auto_sync = (
(homedir, lambda: repos_set_conf("rsync", auto_sync="no")),
(homedir, cmds["emerge"] + ("--sync",)),
(homedir, lambda: self.assertFalse(os.path.exists(
os.path.join(repo.location, "dev-libs", "A")
), "dev-libs/A found, expected missing")),
(homedir, lambda: repos_set_conf("rsync", auto_sync="yes")),
)
rename_repo = (
(homedir, lambda: os.rename(repo.location,
repo.location + "_sync")),
)
rsync_opts_repos = (
(homedir, alter_ebuild),
(homedir, lambda: repos_set_conf("rsync", None,
"sync-rsync-extra-opts = --backup --backup-dir=%s" %
_shell_quote(repo.location + "_back"))),
(homedir, cmds['emerge'] + ("--sync",)),
(homedir, lambda: self.assertTrue(os.path.exists(
repo.location + "_back"))),
(homedir, lambda: shutil.rmtree(repo.location + "_back")),
(homedir, lambda: repos_set_conf("rsync")),
)
rsync_opts_repos_default = (
(homedir, alter_ebuild),
(homedir, lambda: repos_set_conf("rsync",
"sync-rsync-extra-opts = --backup --backup-dir=%s" %
_shell_quote(repo.location+"_back"))),
(homedir, cmds['emerge'] + ("--sync",)),
(homedir, lambda: self.assertTrue(os.path.exists(repo.location + "_back"))),
(homedir, lambda: shutil.rmtree(repo.location + "_back")),
(homedir, lambda: repos_set_conf("rsync")),
)
rsync_opts_repos_default_ovr = (
(homedir, alter_ebuild),
(homedir, lambda: repos_set_conf("rsync",
"sync-rsync-extra-opts = --backup --backup-dir=%s" %
_shell_quote(repo.location + "_back_nowhere"),
"sync-rsync-extra-opts = --backup --backup-dir=%s" %
_shell_quote(repo.location + "_back"))),
(homedir, cmds['emerge'] + ("--sync",)),
(homedir, lambda: self.assertTrue(os.path.exists(repo.location + "_back"))),
(homedir, lambda: shutil.rmtree(repo.location + "_back")),
(homedir, lambda: repos_set_conf("rsync")),
)
rsync_opts_repos_default_cancel = (
(homedir, alter_ebuild),
(homedir, lambda: repos_set_conf("rsync",
"sync-rsync-extra-opts = --backup --backup-dir=%s" %
_shell_quote(repo.location + "_back_nowhere"),
"sync-rsync-extra-opts = ")),
(homedir, cmds['emerge'] + ("--sync",)),
(homedir, lambda: self.assertFalse(os.path.exists(repo.location + "_back"))),
(homedir, lambda: repos_set_conf("rsync")),
)
delete_repo_location = (
(homedir, lambda: shutil.rmtree(repo.user_location)),
(homedir, lambda: os.mkdir(repo.user_location)),
)
revert_rcu_layout = (
(homedir, lambda: os.rename(repo.user_location, repo.user_location + '.bak')),
(homedir, lambda: os.rename(os.path.realpath(repo.user_location + '.bak'), repo.user_location)),
(homedir, lambda: os.unlink(repo.user_location + '.bak')),
(homedir, lambda: shutil.rmtree(repo.user_location + '_rcu_storedir')),
)
upstream_git_commit = (
(
repo.location + "_sync",
git_cmd + ('commit', '--allow-empty', '-m', 'test empty commit'),
),
(
repo.location + "_sync",
git_cmd + ('commit', '--allow-empty', '-m', 'test empty commit 2'),
),
)
delete_sync_repo = (
(homedir, lambda: shutil.rmtree(
repo.location + "_sync")),
)
git_repo_create = (
(repo.location, git_cmd +
("config", "--global", "user.name", committer_name,)),
(repo.location, git_cmd +
("config", "--global", "user.email", committer_email,)),
(repo.location, git_cmd + ("init-db",)),
(repo.location, git_cmd + ("add", ".")),
(repo.location, git_cmd +
("commit", "-a", "-m", "add whole repo")),
)
sync_type_git = (
(homedir, lambda: repos_set_conf("git")),
)
sync_type_git_shallow = (
(homedir, lambda: repos_set_conf("git", sync_depth=1)),
)
sync_rsync_rcu = (
(homedir, lambda: repos_set_conf("rsync", sync_rcu=True)),
)
pythonpath = os.environ.get("PYTHONPATH")
if pythonpath is not None and not pythonpath.strip():
pythonpath = None
if pythonpath is not None and \
pythonpath.split(":")[0] == PORTAGE_PYM_PATH:
pass
else:
if pythonpath is None:
pythonpath = ""
else:
pythonpath = ":" + pythonpath
pythonpath = PORTAGE_PYM_PATH + pythonpath
env = {
"PORTAGE_OVERRIDE_EPREFIX" : eprefix,
"DISTDIR" : distdir,
"GENTOO_COMMITTER_NAME" : committer_name,
"GENTOO_COMMITTER_EMAIL" : committer_email,
"HOME" : homedir,
"PATH" : os.environ["PATH"],
"PORTAGE_GRPNAME" : os.environ["PORTAGE_GRPNAME"],
"PORTAGE_USERNAME" : os.environ["PORTAGE_USERNAME"],
"PYTHONDONTWRITEBYTECODE" : os.environ.get("PYTHONDONTWRITEBYTECODE", ""),
"PYTHONPATH" : pythonpath,
}
repos_set_conf("rsync")
if os.environ.get("SANDBOX_ON") == "1":
# avoid problems from nested sandbox instances
env["FEATURES"] = "-sandbox -usersandbox"
dirs = [homedir, metadata_dir]
try:
for d in dirs:
ensure_dirs(d)
timestamp_path = os.path.join(metadata_dir, 'timestamp.chk')
with open(timestamp_path, 'w') as f:
f.write(bump_timestamp.timestamp.strftime('%s\n' % TIMESTAMP_FORMAT,))
if debug:
# The subprocess inherits both stdout and stderr, for
# debugging purposes.
stdout = None
else:
# The subprocess inherits stderr so that any warnings
# triggered by python -Wd will be visible.
stdout = subprocess.PIPE
for cwd, cmd in rename_repo + sync_cmds_auto_sync + sync_cmds + \
rsync_opts_repos + rsync_opts_repos_default + \
rsync_opts_repos_default_ovr + rsync_opts_repos_default_cancel + \
bump_timestamp_cmds + sync_rsync_rcu + sync_cmds + revert_rcu_layout + \
delete_repo_location + sync_cmds + sync_cmds + \
bump_timestamp_cmds + sync_cmds + revert_rcu_layout + \
delete_sync_repo + git_repo_create + sync_type_git + \
rename_repo + sync_cmds + upstream_git_commit + sync_cmds + \
sync_type_git_shallow + upstream_git_commit + sync_cmds:
if hasattr(cmd, '__call__'):
cmd()
continue
abs_cwd = os.path.join(repo.location, cwd)
proc = subprocess.Popen(cmd,
cwd=abs_cwd, env=env, stdout=stdout)
if debug:
proc.wait()
else:
output = proc.stdout.readlines()
proc.wait()
proc.stdout.close()
if proc.returncode != os.EX_OK:
for line in output:
sys.stderr.write(_unicode_decode(line))
self.assertEqual(os.EX_OK, proc.returncode,
"%s failed in %s" % (cmd, cwd,))
finally:
playground.cleanup()