def unpackinfo(self, mydest): """Unpacks all the files from the dataSegment into 'mydest'.""" if not self.scan(): return 0 mydest = normalize_path(mydest) + os.sep a = open(_unicode_encode(self.file, encoding=_encodings['fs'], errors='strict'), 'rb') if not os.path.exists(mydest): os.makedirs(mydest) startpos = 0 while ((startpos + 8) < self.indexsize): namelen = decodeint(self.index[startpos:startpos + 4]) datapos = decodeint(self.index[startpos + 4 + namelen:startpos + 8 + namelen]) datalen = decodeint(self.index[startpos + 8 + namelen:startpos + 12 + namelen]) myname = self.index[startpos + 4:startpos + 4 + namelen] myname = _unicode_decode(myname, encoding=_encodings['repo.content'], errors='replace') filename = os.path.join(mydest, myname.lstrip(os.sep)) filename = normalize_path(filename) if not filename.startswith(mydest): # myname contains invalid ../ component(s) continue dirname = os.path.dirname(filename) if dirname: if not os.path.exists(dirname): os.makedirs(dirname) mydat = open(_unicode_encode(filename, encoding=_encodings['fs'], errors='strict'), 'wb') a.seek(self.datapos + datapos) mydat.write(a.read(datalen)) mydat.close() startpos = startpos + namelen + 12 a.close() return 1
def xpand(myid,mydest):
myindex=myid[0]
mydata=myid[1]
try:
origdir=os.getcwd()
except SystemExit as e:
raise
except:
os.chdir("/")
origdir="/"
os.chdir(mydest)
myindexlen=len(myindex)
startpos=0
while ((startpos+8)<myindexlen):
namelen=decodeint(myindex[startpos:startpos+4])
datapos=decodeint(myindex[startpos+4+namelen:startpos+8+namelen]);
datalen=decodeint(myindex[startpos+8+namelen:startpos+12+namelen]);
myname=myindex[startpos+4:startpos+4+namelen]
dirname=os.path.dirname(myname)
if dirname:
if not os.path.exists(dirname):
os.makedirs(dirname)
mydat = open(_unicode_encode(myname,
encoding=_encodings['fs'], errors='strict'), 'wb')
mydat.write(mydata[datapos:datapos+datalen])
mydat.close()
startpos=startpos+namelen+12
os.chdir(origdir)
def _file_archive_ensure_dir(parent_dir): """ Ensure that the parent directory for an archive exists. If a file exists where a directory is needed, then rename it (see bug 256376). @param parent_dir: path of parent directory @type parent_dir: str """ for parent in iter_parents(parent_dir): # Use lstat because a symlink to a directory might point # to a directory outside of the config archive, making # it an unsuitable parent. try: parent_st = os.lstat(parent) except OSError: pass else: if not stat.S_ISDIR(parent_st.st_mode): _file_archive_rotate(parent) break try: os.makedirs(parent_dir) except OSError: pass
def xpand(myid, mydest): mydest = normalize_path(mydest) + os.sep myindex = myid[0] mydata = myid[1] myindexlen = len(myindex) startpos = 0 while ((startpos + 8) < myindexlen): namelen = decodeint(myindex[startpos:startpos + 4]) datapos = decodeint(myindex[startpos + 4 + namelen:startpos + 8 + namelen]) datalen = decodeint(myindex[startpos + 8 + namelen:startpos + 12 + namelen]) myname = myindex[startpos + 4:startpos + 4 + namelen] myname = _unicode_decode(myname, encoding=_encodings['repo.content'], errors='replace') filename = os.path.join(mydest, myname.lstrip(os.sep)) filename = normalize_path(filename) if not filename.startswith(mydest): # myname contains invalid ../ component(s) continue dirname = os.path.dirname(filename) if dirname: if not os.path.exists(dirname): os.makedirs(dirname) mydat = open(_unicode_encode(filename, encoding=_encodings['fs'], errors='strict'), 'wb') mydat.write(mydata[datapos:datapos + datalen]) mydat.close() startpos = startpos + namelen + 12
def dodir(path): try: os.makedirs(path, 0o755) except OSError: if not os.path.isdir(path): raise os.chmod(path, 0o755)
def ensure_dirs(dir_path, *args, **kwargs):
"""Create a directory and call apply_permissions.
Returns True if a directory is created or the permissions needed to be
modified, and False otherwise."""
created_dir = False
try:
os.makedirs(dir_path)
created_dir = True
except OSError as oe:
func_call = "makedirs('%s')" % dir_path
if oe.errno in (errno.EEXIST, errno.EISDIR):
pass
else:
if os.path.isdir(dir_path):
# NOTE: DragonFly raises EPERM for makedir('/')
# and that is supposed to be ignored here.
pass
elif oe.errno == errno.EPERM:
raise OperationNotPermitted(func_call)
elif oe.errno == errno.EACCES:
raise PermissionDenied(func_call)
elif oe.errno == errno.EROFS:
raise ReadOnlyFileSystem(func_call)
else:
raise
perms_modified = apply_permissions(dir_path, *args, **kwargs)
return created_dir or perms_modified
def __init__(self, ebuilds={}, installed={}, profile={}, repo_configs={}, \
user_config={}, sets={}, world=[], world_sets=[], distfiles={}, debug=False):
"""
ebuilds: cpv -> metadata mapping simulating available ebuilds.
installed: cpv -> metadata mapping simulating installed packages.
If a metadata key is missing, it gets a default value.
profile: settings defined by the profile.
"""
self.debug = debug
self.eprefix = normalize_path(tempfile.mkdtemp())
self.eroot = self.eprefix + os.sep
self.distdir = os.path.join(self.eroot, "var", "portage", "distfiles")
self.portdir = os.path.join(self.eroot, "usr/portage")
self.vdbdir = os.path.join(self.eroot, "var/db/pkg")
os.makedirs(self.portdir)
os.makedirs(self.vdbdir)
if not debug:
portage.util.noiselimit = -2
self.repo_dirs = {}
#Make sure the main repo is always created
self._get_repo_dir("test_repo")
self._create_distfiles(distfiles)
self._create_ebuilds(ebuilds)
self._create_installed(installed)
self._create_profile(ebuilds, installed, profile, repo_configs, user_config, sets)
self._create_world(world, world_sets)
self.settings, self.trees = self._load_config()
self._create_ebuild_manifests(ebuilds)
portage.util.noiselimit = 0
def _create_installed(self, installed):
for cpv in installed:
a = Atom("=" + cpv, allow_repo=True)
repo = a.repo
if repo is None:
repo = "test_repo"
vdb_pkg_dir = os.path.join(self.vdbdir, a.cpv)
try:
os.makedirs(vdb_pkg_dir)
except os.error:
pass
metadata = installed[cpv].copy()
metadata.setdefault("SLOT", "0")
metadata.setdefault("BUILD_TIME", "0")
metadata.setdefault("COUNTER", "0")
metadata.setdefault("KEYWORDS", "~x86")
unknown_keys = set(metadata).difference(
portage.dbapi.dbapi._known_keys)
unknown_keys.discard("BUILD_TIME")
unknown_keys.discard("COUNTER")
unknown_keys.discard("repository")
unknown_keys.discard("USE")
if unknown_keys:
raise ValueError("metadata of installed '%s' contains unknown keys: %s" %
(cpv, sorted(unknown_keys)))
metadata["repository"] = repo
for k, v in metadata.items():
with open(os.path.join(vdb_pkg_dir, k), "w") as f:
f.write("%s\n" % v)
def new(self, **kwargs):
'''Do the initial clone of the repository'''
if kwargs:
self._kwargs(kwargs)
emerge_config = self.options.get('emerge_config', None)
portdb = self.options.get('portdb', None)
try:
if not os.path.exists(self.repo.location):
os.makedirs(self.repo.location)
self.logger(self.xterm_titles,
'Created new directory %s' % self.repo.location)
except IOError:
return (1, False)
msg = ">>> Cloning git repository from upstream into %s..." % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
sync_uri = self.repo.sync_uri
if sync_uri.startswith("file://"):
sync_uri = sync_uri[6:]
exitcode = portage.process.spawn_bash("cd %s ; %s clone %s ." % \
(portage._shell_quote(self.repo.location),
self.bin_command,
portage._shell_quote(sync_uri)),
**portage._native_kwargs(self.spawn_kwargs))
if exitcode != os.EX_OK:
msg = "!!! git clone error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
msg = ">>> Git clone successful"
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
return (os.EX_OK, True)
def new(self, **kwargs):
'''Do the initial clone of the repository'''
if kwargs:
self._kwargs(kwargs)
try:
if not os.path.exists(self.repo.location):
os.makedirs(self.repo.location)
self.logger(self.xterm_titles,
'Created new directory %s' % self.repo.location)
except IOError:
return (1, False)
sync_uri = self.repo.sync_uri
if sync_uri.startswith("file://"):
sync_uri = sync_uri[6:]
git_cmd_opts = ""
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.sync_depth is not None:
git_cmd_opts += " --depth %d" % self.repo.sync_depth
git_cmd = "%s clone%s %s ." % (self.bin_command, git_cmd_opts,
portage._shell_quote(sync_uri))
writemsg_level(git_cmd + "\n")
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**portage._native_kwargs(self.spawn_kwargs))
if exitcode != os.EX_OK:
msg = "!!! git clone error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
return (os.EX_OK, True)
def setUp(self):
super(RepomanEchangelogTestCase, self).setUp()
self.tmpdir = tempfile.mkdtemp(prefix='repoman.echangelog.')
self.skel_changelog = os.path.join(self.tmpdir, 'skel.ChangeLog')
skel = [
'# ChangeLog for <CATEGORY>/<PACKAGE_NAME>\n',
'# Copyright 1999-2000 Gentoo Foundation; Distributed under the GPL v2\n',
'# $Header: $\n'
]
self._writelines(self.skel_changelog, skel)
self.cat = 'mycat'
self.pkg = 'mypkg'
self.pkgdir = os.path.join(self.tmpdir, self.cat, self.pkg)
os.makedirs(self.pkgdir)
self.header_pkg = '# ChangeLog for %s/%s\n' % (self.cat, self.pkg)
self.header_copyright = '# Copyright 1999-%s Gentoo Foundation; Distributed under the GPL v2\n' % \
time.strftime('%Y', time.gmtime())
self.header_cvs = '# $Header: $\n'
self.changelog = os.path.join(self.pkgdir, 'ChangeLog')
self.user = '******'
def testSetCpv(self):
"""
Test the clone via constructor.
"""
ebuilds = {
"dev-libs/A-1": {"IUSE": "static-libs"},
"dev-libs/B-1": {"IUSE": "static-libs"},
}
env_files = {
"A" : ("USE=\"static-libs\"",)
}
package_env = (
"dev-libs/A A",
)
eprefix = normalize_path(tempfile.mkdtemp())
playground = None
try:
user_config_dir = os.path.join(eprefix, USER_CONFIG_PATH)
os.makedirs(user_config_dir)
with io.open(os.path.join(user_config_dir, "package.env"),
mode='w', encoding=_encodings['content']) as f:
for line in package_env:
f.write(line + "\n")
env_dir = os.path.join(user_config_dir, "env")
os.makedirs(env_dir)
for k, v in env_files.items():
with io.open(os.path.join(env_dir, k), mode='w',
encoding=_encodings['content']) as f:
for line in v:
f.write(line + "\n")
playground = ResolverPlayground(eprefix=eprefix, ebuilds=ebuilds)
settings = config(clone=playground.settings)
result = playground.run(["=dev-libs/A-1"])
pkg, existing_node = result.depgraph._select_package(
playground.eroot, Atom("=dev-libs/A-1"))
settings.setcpv(pkg)
self.assertTrue("static-libs" in
settings["PORTAGE_USE"].split())
# Test bug #522362, where a USE=static-libs package.env
# setting leaked from one setcpv call to the next.
pkg, existing_node = result.depgraph._select_package(
playground.eroot, Atom("=dev-libs/B-1"))
settings.setcpv(pkg)
self.assertTrue("static-libs" not in
settings["PORTAGE_USE"].split())
finally:
if playground is None:
shutil.rmtree(eprefix)
else:
playground.cleanup()
def __init__(self, ebuilds={}, installed={}, profile={}, user_config={}, sets={}, world=[], debug=False):
"""
ebuilds: cpv -> metadata mapping simulating avaiable ebuilds.
installed: cpv -> metadata mapping simulating installed packages.
If a metadata key is missing, it gets a default value.
profile: settings defined by the profile.
"""
self.debug = debug
self.root = "/"
self.eprefix = tempfile.mkdtemp()
self.eroot = self.root + self.eprefix.lstrip(os.sep) + os.sep
self.portdir = os.path.join(self.eroot, "usr/portage")
self.vdbdir = os.path.join(self.eroot, "var/db/pkg")
os.makedirs(self.portdir)
os.makedirs(self.vdbdir)
if not debug:
portage.util.noiselimit = -2
self._create_ebuilds(ebuilds)
self._create_installed(installed)
self._create_profile(ebuilds, installed, profile, user_config, sets)
self._create_world(world)
self.settings, self.trees = self._load_config()
self._create_ebuild_manifests(ebuilds)
portage.util.noiselimit = 0
def save_cache(logger, to_save={}, temp_path=DEFAULTS['DEFAULT_TMP_DIR']):
''' Tries to store caching information.
@param logger
@param to_save have to be dict with keys:
libraries, la_libraries, libraries_links and binaries
'''
if not os.path.exists(temp_path):
os.makedirs(temp_path)
try:
_file = open(_unicode_encode(os.path.join(temp_path, 'timestamp'),
encoding=_encodings['fs']), mode='w', encoding=_encodings['content'])
_file.write(_unicode(int(time.time())))
_file.close()
for key,val in to_save.items():
_file = open(_unicode_encode(os.path.join(temp_path, key),
encoding=_encodings['fs']), mode='w',
encoding=_encodings['content'])
for line in val:
_file.write(line + '\n')
_file.close()
except Exception as ex:
logger.warning('\t' + red('Could not save cache: %s' %str(ex)))
def testDoebuildSpawn(self):
playground = ResolverPlayground()
try:
settings = config(clone=playground.settings)
cpv = 'sys-apps/portage-2.1'
metadata = {
'EAPI' : '2',
'INHERITED' : 'python eutils',
'IUSE' : 'build doc epydoc python3 selinux',
'LICENSE' : 'GPL-2',
'PROVIDE' : 'virtual/portage',
'RDEPEND' : '>=app-shells/bash-3.2_p17 >=dev-lang/python-2.6',
'SLOT' : '0',
}
root_config = playground.trees[playground.eroot]['root_config']
pkg = Package(built=False, cpv=cpv, installed=False,
metadata=metadata, root_config=root_config,
type_name='ebuild')
settings.setcpv(pkg)
settings['PORTAGE_PYTHON'] = _python_interpreter
settings['PORTAGE_BUILDDIR'] = os.path.join(
settings['PORTAGE_TMPDIR'], cpv)
settings['T'] = os.path.join(
settings['PORTAGE_BUILDDIR'], 'temp')
for x in ('PORTAGE_BUILDDIR', 'T'):
os.makedirs(settings[x])
# Create a fake environment, to pretend as if the ebuild
# has been sourced already.
open(os.path.join(settings['T'], 'environment'), 'wb').close()
scheduler = PollScheduler().sched_iface
for phase in ('_internal_test',):
# Test EbuildSpawnProcess by calling doebuild.spawn() with
# returnpid=False. This case is no longer used by portage
# internals since EbuildPhase is used instead and that passes
# returnpid=True to doebuild.spawn().
rval = doebuild_spawn("%s %s" % (_shell_quote(
os.path.join(settings["PORTAGE_BIN_PATH"],
os.path.basename(EBUILD_SH_BINARY))), phase),
settings, free=1)
self.assertEqual(rval, os.EX_OK)
ebuild_phase = EbuildPhase(background=False,
phase=phase, scheduler=scheduler,
settings=settings)
ebuild_phase.start()
ebuild_phase.wait()
self.assertEqual(ebuild_phase.returncode, os.EX_OK)
ebuild_phase = MiscFunctionsProcess(background=False,
commands=['success_hooks'],
scheduler=scheduler, settings=settings)
ebuild_phase.start()
ebuild_phase.wait()
self.assertEqual(ebuild_phase.returncode, os.EX_OK)
finally:
playground.cleanup()
def new(self, **kwargs):
'''Do the initial clone of the repository'''
if kwargs:
self._kwargs(kwargs)
if not self.has_bin:
return (1, False)
try:
if not os.path.exists(self.repo.location):
os.makedirs(self.repo.location)
self.logger(self.xterm_titles,
'Created new directory %s' % self.repo.location)
except IOError:
return (1, False)
sync_uri = self.repo.sync_uri
if sync_uri.startswith("file://"):
sync_uri = sync_uri[7:]
git_cmd_opts = ""
if self.repo.module_specific_options.get('sync-git-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-env'])
env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(env)
if self.repo.module_specific_options.get('sync-git-clone-env'):
shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-clone-env'])
clone_env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
self.spawn_kwargs['env'].update(clone_env)
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.clone_depth is not None:
if self.repo.clone_depth != 0:
git_cmd_opts += " --depth %d" % self.repo.clone_depth
elif self.repo.sync_depth is not None:
if self.repo.sync_depth != 0:
git_cmd_opts += " --depth %d" % self.repo.sync_depth
else:
# default
git_cmd_opts += " --depth 1"
if self.repo.module_specific_options.get('sync-git-clone-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options['sync-git-clone-extra-opts']
git_cmd = "%s clone%s %s ." % (self.bin_command, git_cmd_opts,
portage._shell_quote(sync_uri))
writemsg_level(git_cmd + "\n")
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), git_cmd),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! git clone error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
if not self.verify_head():
return (1, False)
return (os.EX_OK, True)
def testFakedbapi(self):
packages = (
("sys-apps/portage-2.1.10", {
"EAPI" : "2",
"IUSE" : "ipc doc",
"repository" : "gentoo",
"SLOT" : "0",
"USE" : "ipc missing-iuse",
}),
("virtual/package-manager-0", {
"EAPI" : "0",
"repository" : "gentoo",
"SLOT" : "0",
}),
)
match_tests = (
("sys-apps/portage:0[ipc]", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0[-ipc]", []),
("sys-apps/portage:0[doc]", []),
("sys-apps/portage:0[-doc]", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0[missing-iuse]", []),
("sys-apps/portage:0[-missing-iuse]", []),
("sys-apps/portage:0::gentoo[ipc]", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0::multilib[ipc]", []),
("virtual/package-manager", ["virtual/package-manager-0"]),
)
tempdir = tempfile.mkdtemp()
try:
test_repo = os.path.join(tempdir, "var", "repositories", "test_repo")
os.makedirs(os.path.join(test_repo, "profiles"))
with open(os.path.join(test_repo, "profiles", "repo_name"), "w") as f:
f.write("test_repo")
env = {
"PORTAGE_REPOSITORIES": "[DEFAULT]\nmain-repo = test_repo\n[test_repo]\nlocation = %s" % test_repo
}
# Tests may override portage.const.EPREFIX in order to
# simulate a prefix installation. It's reasonable to do
# this because tests should be self-contained such that
# the "real" value of portage.const.EPREFIX is entirely
# irrelevant (see bug #492932).
portage.const.EPREFIX = tempdir
fakedb = fakedbapi(settings=config(config_profile_path="",
env=env, eprefix=tempdir))
for cpv, metadata in packages:
fakedb.cpv_inject(cpv, metadata=metadata)
for atom, expected_result in match_tests:
result = fakedb.match(atom)
self.assertEqual(fakedb.match(atom), expected_result,
"fakedb.match('%s') = %s != %s" %
(atom, result, expected_result))
finally:
shutil.rmtree(tempdir)
def _create_installed(self, installed):
for cpv in installed:
a = Atom("=" + cpv, allow_repo=True)
repo = a.repo
if repo is None:
repo = "test_repo"
vdb_pkg_dir = os.path.join(self.vdbdir, a.cpv)
try:
os.makedirs(vdb_pkg_dir)
except os.error:
pass
metadata = installed[cpv].copy()
eapi = metadata.pop("EAPI", 0)
lic = metadata.pop("LICENSE", "")
properties = metadata.pop("PROPERTIES", "")
slot = metadata.pop("SLOT", 0)
build_time = metadata.pop("BUILD_TIME", "0")
keywords = metadata.pop("KEYWORDS", "~x86")
iuse = metadata.pop("IUSE", "")
use = metadata.pop("USE", "")
provide = metadata.pop("PROVIDE", None)
depend = metadata.pop("DEPEND", "")
rdepend = metadata.pop("RDEPEND", None)
pdepend = metadata.pop("PDEPEND", None)
required_use = metadata.pop("REQUIRED_USE", None)
if metadata:
raise ValueError("metadata of installed '%s' contains unknown keys: %s" % (cpv, metadata.keys()))
def write_key(key, value):
f = open(os.path.join(vdb_pkg_dir, key), "w")
f.write(str(value) + "\n")
f.close()
write_key("EAPI", eapi)
write_key("BUILD_TIME", build_time)
write_key("COUNTER", "0")
write_key("LICENSE", lic)
write_key("PROPERTIES", properties)
write_key("SLOT", slot)
write_key("LICENSE", lic)
write_key("PROPERTIES", properties)
write_key("repository", repo)
write_key("KEYWORDS", keywords)
write_key("IUSE", iuse)
write_key("USE", use)
if provide is not None:
write_key("PROVIDE", provide)
write_key("DEPEND", depend)
if rdepend is not None:
write_key("RDEPEND", rdepend)
if pdepend is not None:
write_key("PDEPEND", pdepend)
if required_use is not None:
write_key("REQUIRED_USE", required_use)
def new(self, **kwargs):
if kwargs:
self._kwargs(kwargs)
try:
if not os.path.exists(self.repo.location):
os.makedirs(self.repo.location)
self.logger(self.self.xterm_titles, "Created New Directory %s " % self.repo.location)
except IOError:
return (1, False)
return self.update()
def _create_world(self, world):
#Create /var/lib/portage/world
var_lib_portage = os.path.join(self.eroot, "var", "lib", "portage")
os.makedirs(var_lib_portage)
world_file = os.path.join(var_lib_portage, "world")
f = open(world_file, "w")
for atom in world:
f.write("%s\n" % atom)
f.close()
def rcs_archive(archive, curconf, newconf, mrgconf): """Archive existing config in rcs (on trunk). Then, if mrgconf is specified and an old branch version exists, merge the user's changes and the distributed changes and put the result into mrgconf. Lastly, if newconf was specified, leave it in the archive dir with a .dist.new suffix along with the last 1.1.1 branch version with a .dist suffix.""" try: os.makedirs(os.path.dirname(archive)) except OSError: pass try: curconf_st = os.lstat(curconf) except OSError: curconf_st = None if curconf_st is not None and \ (stat.S_ISREG(curconf_st.st_mode) or stat.S_ISLNK(curconf_st.st_mode)): _archive_copy(curconf_st, curconf, archive) if os.path.lexists(archive + ',v'): os.system(RCS_LOCK + ' ' + archive) os.system(RCS_PUT + ' ' + archive) ret = 0 mystat = None if newconf: try: mystat = os.lstat(newconf) except OSError: pass if mystat is not None and \ (stat.S_ISREG(mystat.st_mode) or stat.S_ISLNK(mystat.st_mode)): os.system(RCS_GET + ' -r' + RCS_BRANCH + ' ' + archive) has_branch = os.path.lexists(archive) if has_branch: os.rename(archive, archive + '.dist') _archive_copy(mystat, newconf, archive) if has_branch: if mrgconf and os.path.isfile(archive) and \ os.path.isfile(mrgconf): # This puts the results of the merge into mrgconf. ret = os.system(RCS_MERGE % (archive, mrgconf)) os.chmod(mrgconf, mystat.st_mode) os.chown(mrgconf, mystat.st_uid, mystat.st_gid) os.rename(archive, archive + '.dist.new') return ret
def decompose(self, datadir, cleanup=1): """Alias for unpackinfo() --- Complement to recompose() but optionally deletes the destination directory. Extracts the xpak from the tbz2 into the directory provided. Raises IOError if scan() fails. Returns result of upackinfo().""" if not self.scan(): raise IOError if cleanup: self.cleanup(datadir) if not os.path.exists(datadir): os.makedirs(datadir) return self.unpackinfo(datadir)
def __init__(self, ebuilds={}, binpkgs={}, installed={}, profile={}, repo_configs={}, \
user_config={}, sets={}, world=[], world_sets=[], distfiles={},
eprefix=None, targetroot=False, debug=False):
"""
ebuilds: cpv -> metadata mapping simulating available ebuilds.
installed: cpv -> metadata mapping simulating installed packages.
If a metadata key is missing, it gets a default value.
profile: settings defined by the profile.
"""
self.debug = debug
if eprefix is None:
self.eprefix = normalize_path(tempfile.mkdtemp())
else:
self.eprefix = normalize_path(eprefix)
# Tests may override portage.const.EPREFIX in order to
# simulate a prefix installation. It's reasonable to do
# this because tests should be self-contained such that
# the "real" value of portage.const.EPREFIX is entirely
# irrelevant (see bug #492932).
portage.const.EPREFIX = self.eprefix.rstrip(os.sep)
self.eroot = self.eprefix + os.sep
if targetroot:
self.target_root = os.path.join(self.eroot, 'target_root')
else:
self.target_root = os.sep
self.distdir = os.path.join(self.eroot, "var", "portage", "distfiles")
self.pkgdir = os.path.join(self.eprefix, "pkgdir")
self.vdbdir = os.path.join(self.eroot, "var/db/pkg")
os.makedirs(self.vdbdir)
if not debug:
portage.util.noiselimit = -2
self._repositories = {}
#Make sure the main repo is always created
self._get_repo_dir("test_repo")
self._create_distfiles(distfiles)
self._create_ebuilds(ebuilds)
self._create_binpkgs(binpkgs)
self._create_installed(installed)
self._create_profile(ebuilds, installed, profile, repo_configs, user_config, sets)
self._create_world(world, world_sets)
self.settings, self.trees = self._load_config()
self._create_ebuild_manifests(ebuilds)
portage.util.noiselimit = 0
def makedirs(dir_path):
try:
os.makedirs(dir_path)
except OSError as oe:
if errno.EEXIST == oe.errno:
pass
elif errno.EPERM == oe.errno:
writemsg("%s\n" % oe, noiselevel=-1)
writemsg(_("Operation Not Permitted: makedirs('%s')\n") % dir_path, noiselevel=-1)
return False
else:
raise
return True
def test_gpkg_get_metadata_url(self):
if sys.version_info.major < 3:
self.skipTest("Not support Python 2")
if sys.version_info.major == 3 and sys.version_info.minor <= 6:
self.skipTest("http server not support change root dir")
playground = ResolverPlayground(
user_config={
"make.conf": (
'BINPKG_COMPRESS="gzip"',
'FEATURES="${FEATURES} -binpkg-signing '
'-binpkg-request-signature"',
),
})
tmpdir = tempfile.mkdtemp()
try:
settings = playground.settings
for _ in range(0, 5):
port = random.randint(30000, 60000)
try:
server = self.start_http_server(tmpdir, port)
except OSError:
continue
break
orig_full_path = os.path.join(tmpdir, "orig/")
os.makedirs(orig_full_path)
with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
test_file.write(urandom(1048576))
gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
test_gpkg = gpkg(settings, "test", gpkg_file_loc)
meta = {
"test1": b"{abcdefghijklmnopqrstuvwxyz, 1234567890}",
"test2": urandom(102400),
}
test_gpkg.compress(os.path.join(tmpdir, "orig"), meta)
meta_from_url = test_gpkg.get_metadata_url("http://127.0.0.1:" +
str(port) +
"/test.gpkg.tar")
self.assertEqual(meta, meta_from_url)
finally:
shutil.rmtree(tmpdir)
playground.cleanup()
def makedirs(dir_path):
try:
os.makedirs(dir_path)
except OSError as oe:
if errno.EEXIST == oe.errno:
pass
elif errno.EPERM == oe.errno:
writemsg("%s\n" % oe, noiselevel=-1)
writemsg(_("Operation Not Permitted: makedirs('%s')\n") % \
dir_path, noiselevel=-1)
return False
else:
raise
return True
def _create_installed(self, installed):
for cpv in installed:
a = Atom("=" + cpv, allow_repo=True)
repo = a.repo
if repo is None:
repo = "test_repo"
vdb_pkg_dir = os.path.join(self.vdbdir, a.cpv)
try:
os.makedirs(vdb_pkg_dir)
except os.error:
pass
metadata = installed[cpv].copy()
metadata.setdefault("SLOT", "0")
metadata.setdefault("BUILD_TIME", "0")
metadata.setdefault("COUNTER", "0")
metadata.setdefault("KEYWORDS", "~x86")
unknown_keys = set(metadata).difference(
portage.dbapi.dbapi._known_keys)
unknown_keys.discard("BUILD_TIME")
unknown_keys.discard("BUILD_ID")
unknown_keys.discard("COUNTER")
unknown_keys.discard("repository")
unknown_keys.discard("USE")
unknown_keys.discard("PROVIDES")
unknown_keys.discard("REQUIRES")
if unknown_keys:
raise ValueError(
"metadata of installed '%s' contains unknown keys: %s" %
(cpv, sorted(unknown_keys)))
metadata["repository"] = repo
for k, v in metadata.items():
with open(os.path.join(vdb_pkg_dir, k), "w") as f:
f.write("%s\n" % v)
ebuild_path = os.path.join(vdb_pkg_dir,
a.cpv.split("/")[1] + ".ebuild")
with open(ebuild_path, "w") as f:
f.write('EAPI="%s"\n' % metadata.pop("EAPI", "0"))
for k, v in metadata.items():
f.write('%s="%s"\n' % (k, v))
env_path = os.path.join(vdb_pkg_dir, "environment.bz2")
with bz2.BZ2File(env_path, mode="w") as f:
with open(ebuild_path, "rb") as inputfile:
f.write(inputfile.read())
def test_gpkg_incorrect_checksum(self):
if sys.version_info.major < 3:
self.skipTest("Not support Python 2")
playground = ResolverPlayground(
user_config={
"make.conf": (
'FEATURES="${FEATURES} -binpkg-signing '
'-binpkg-request-signature -gpg-keepalive"',
),
}
)
tmpdir = tempfile.mkdtemp()
try:
settings = playground.settings
orig_full_path = os.path.join(tmpdir, "orig/")
os.makedirs(orig_full_path)
data = urandom(1048576)
with open(os.path.join(orig_full_path, "data"), "wb") as f:
f.write(data)
binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
binpkg_1.compress(orig_full_path, {})
with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
with tarfile.open(
os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
) as tar_2:
for f in tar_1.getmembers():
if f.name == "Manifest":
data = io.BytesIO(tar_1.extractfile(f).read())
data_view = data.getbuffer()
data_view[-16:] = b"20a6d80ab0320fh9"
del data_view
tar_2.addfile(f, data)
data.close()
else:
tar_2.addfile(f, tar_1.extractfile(f))
binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
self.assertRaises(
DigestException, binpkg_2.decompress, os.path.join(tmpdir, "test")
)
finally:
shutil.rmtree(tmpdir)
playground.cleanup()
def _create_world(self, world, world_sets):
#Create /var/lib/portage/world
var_lib_portage = os.path.join(self.eroot, "var", "lib", "portage")
os.makedirs(var_lib_portage)
world_file = os.path.join(var_lib_portage, "world")
world_set_file = os.path.join(var_lib_portage, "world_sets")
with open(world_file, "w") as f:
for atom in world:
f.write("%s\n" % atom)
with open(world_set_file, "w") as f:
for atom in world_sets:
f.write("%s\n" % atom)
def _create_world(self, world, world_sets):
# Create /var/lib/portage/world
var_lib_portage = os.path.join(self.eroot, "var", "lib", "portage")
os.makedirs(var_lib_portage)
world_file = os.path.join(var_lib_portage, "world")
world_set_file = os.path.join(var_lib_portage, "world_sets")
with open(world_file, "w") as f:
for atom in world:
f.write("%s\n" % atom)
with open(world_set_file, "w") as f:
for atom in world_sets:
f.write("%s\n" % atom)
def testFakedbapi(self):
packages = (
("sys-apps/portage-2.1.10", {
"EAPI" : "2",
"IUSE" : "ipc doc",
"repository" : "gentoo",
"SLOT" : "0",
"USE" : "ipc missing-iuse",
}),
("virtual/package-manager-0", {
"EAPI" : "0",
"repository" : "gentoo",
"SLOT" : "0",
}),
)
match_tests = (
("sys-apps/portage:0[ipc]", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0[-ipc]", []),
("sys-apps/portage:0[doc]", []),
("sys-apps/portage:0[-doc]", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0[missing-iuse]", []),
("sys-apps/portage:0[-missing-iuse]", []),
("sys-apps/portage:0::gentoo[ipc]", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0::multilib[ipc]", []),
("virtual/package-manager", ["virtual/package-manager-0"]),
)
tempdir = tempfile.mkdtemp()
try:
portdir = os.path.join(tempdir, "usr/portage")
os.makedirs(portdir)
env = {
"PORTDIR": portdir,
}
fakedb = fakedbapi(settings=config(config_profile_path="",
env=env, eprefix=tempdir))
for cpv, metadata in packages:
fakedb.cpv_inject(cpv, metadata=metadata)
for atom, expected_result in match_tests:
result = fakedb.match(atom)
self.assertEqual(fakedb.match(atom), expected_result,
"fakedb.match('%s') = %s != %s" %
(atom, result, expected_result))
finally:
shutil.rmtree(tempdir)
def test_gpkg_missing_signature(self):
if sys.version_info.major < 3:
self.skipTest("Not support Python 2")
playground = ResolverPlayground(
user_config={
"make.conf": (
'FEATURES="${FEATURES} binpkg-signing '
'binpkg-request-signature"',
'BINPKG_FORMAT="gpkg"',
),
})
tmpdir = tempfile.mkdtemp()
try:
settings = playground.settings
gpg = GPG(settings)
gpg.unlock()
orig_full_path = os.path.join(tmpdir, "orig/")
os.makedirs(orig_full_path)
data = urandom(1048576)
with open(os.path.join(orig_full_path, "data"), "wb") as f:
f.write(data)
binpkg_1 = gpkg(settings, "test",
os.path.join(tmpdir, "test-1.gpkg.tar"))
binpkg_1.compress(orig_full_path, {})
with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"),
"r") as tar_1:
with tarfile.open(os.path.join(tmpdir, "test-2.gpkg.tar"),
"w") as tar_2:
for f in tar_1.getmembers():
if f.name.endswith(".sig"):
pass
else:
tar_2.addfile(f, tar_1.extractfile(f))
binpkg_2 = gpkg(settings, "test",
os.path.join(tmpdir, "test-2.gpkg.tar"))
self.assertRaises(MissingSignature, binpkg_2.decompress,
os.path.join(tmpdir, "test"))
finally:
shutil.rmtree(tmpdir)
playground.cleanup()
def test_gpkg_extra_files(self):
if sys.version_info.major < 3:
self.skipTest("Not support Python 2")
playground = ResolverPlayground(
user_config={
"make.conf": (
'FEATURES="${FEATURES} -binpkg-signing '
'-binpkg-request-signature -gpg-keepalive"',
),
}
)
tmpdir = tempfile.mkdtemp()
try:
settings = playground.settings
orig_full_path = os.path.join(tmpdir, "orig/")
os.makedirs(orig_full_path)
data = urandom(1048576)
with open(os.path.join(orig_full_path, "data"), "wb") as f:
f.write(data)
binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
binpkg_1.compress(orig_full_path, {})
with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
with tarfile.open(
os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
) as tar_2:
for f in tar_1.getmembers():
tar_2.addfile(f, tar_1.extractfile(f))
data_tarinfo = tarfile.TarInfo("data2")
data_tarinfo.size = len(data)
data2 = io.BytesIO(data)
tar_2.addfile(data_tarinfo, data2)
data2.close()
binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
self.assertRaises(
DigestException, binpkg_2.decompress, os.path.join(tmpdir, "test")
)
finally:
shutil.rmtree(tmpdir)
playground.cleanup()
def _create_installed(self, installed):
for cpv in installed:
a = Atom("=" + cpv)
vdb_pkg_dir = os.path.join(self.vdbdir, a.cpv)
try:
os.makedirs(vdb_pkg_dir)
except os.error:
pass
metadata = installed[cpv].copy()
eapi = metadata.pop("EAPI", 0)
lic = metadata.pop("LICENSE", "")
properties = metadata.pop("PROPERTIES", "")
slot = metadata.pop("SLOT", 0)
keywords = metadata.pop("KEYWORDS", "~x86")
iuse = metadata.pop("IUSE", "")
use = metadata.pop("USE", "")
depend = metadata.pop("DEPEND", "")
rdepend = metadata.pop("RDEPEND", None)
pdepend = metadata.pop("PDEPEND", None)
required_use = metadata.pop("REQUIRED_USE", None)
if metadata:
raise ValueError(
"metadata of installed '%s' contains unknown keys: %s" %
(cpv, metadata.keys()))
def write_key(key, value):
f = open(os.path.join(vdb_pkg_dir, key), "w")
f.write(str(value) + "\n")
f.close()
write_key("EAPI", eapi)
write_key("LICENSE", lic)
write_key("PROPERTIES", properties)
write_key("SLOT", slot)
write_key("KEYWORDS", keywords)
write_key("IUSE", iuse)
write_key("USE", use)
write_key("DEPEND", depend)
if rdepend is not None:
write_key("RDEPEND", rdepend)
if pdepend is not None:
write_key("PDEPEND", pdepend)
if required_use is not None:
write_key("REQUIRED_USE", required_use)
def testFakedbapi(self):
packages = (
("sys-apps/portage-2.1.10", {
"EAPI": "2",
"IUSE": "ipc doc",
"repository": "gentoo",
"SLOT": "0",
"USE": "ipc missing-iuse",
}),
("virtual/package-manager-0", {
"EAPI": "0",
"repository": "gentoo",
"SLOT": "0",
}),
)
match_tests = (
("sys-apps/portage:0[ipc]", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0[-ipc]", []),
("sys-apps/portage:0[doc]", []),
("sys-apps/portage:0[-doc]", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0[missing-iuse]", []),
("sys-apps/portage:0[-missing-iuse]", []),
("sys-apps/portage:0::gentoo[ipc]", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0::multilib[ipc]", []),
("virtual/package-manager", ["virtual/package-manager-0"]),
)
tempdir = tempfile.mkdtemp()
try:
portdir = os.path.join(tempdir, "usr/portage")
os.makedirs(portdir)
env = {
"PORTDIR": portdir,
}
fakedb = fakedbapi(settings=config(
config_profile_path="", env=env, eprefix=tempdir))
for cpv, metadata in packages:
fakedb.cpv_inject(cpv, metadata=metadata)
for atom, expected_result in match_tests:
self.assertEqual(fakedb.match(atom), expected_result)
finally:
shutil.rmtree(tempdir)
def new(self, **kwargs):
'''Do the initial clone of the repository'''
if kwargs:
self._kwargs(kwargs)
try:
if not os.path.exists(self.repo.location):
os.makedirs(self.repo.location)
self.logger(self.xterm_titles,
'Created new directory %s' % self.repo.location)
except IOError:
return (1, False)
sync_uri = self.repo.sync_uri
if sync_uri.startswith("file://"):
sync_uri = sync_uri[6:]
git_cmd_opts = ""
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.clone_depth is not None:
if self.repo.clone_depth != 0:
git_cmd_opts += " --depth %d" % self.repo.clone_depth
elif self.repo.sync_depth is not None:
if self.repo.sync_depth != 0:
git_cmd_opts += " --depth %d" % self.repo.sync_depth
else:
# default
git_cmd_opts += " --depth 1"
if self.repo.module_specific_options.get('sync-git-clone-extra-opts'):
git_cmd_opts += " %s" % self.repo.module_specific_options[
'sync-git-clone-extra-opts']
git_cmd = "%s clone%s %s ." % (self.bin_command, git_cmd_opts,
portage._shell_quote(sync_uri))
writemsg_level(git_cmd + "\n")
exitcode = portage.process.spawn_bash(
"cd %s ; exec %s" %
(portage._shell_quote(self.repo.location), git_cmd),
**self.spawn_kwargs)
if exitcode != os.EX_OK:
msg = "!!! git clone error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
return (exitcode, False)
return (os.EX_OK, True)
def test_gpkg_untrusted_signature(self):
if sys.version_info.major < 3:
self.skipTest("Not support Python 2")
gpg_test_path = os.environ["PORTAGE_GNUPGHOME"]
playground = ResolverPlayground(
user_config={
"make.conf": (
'FEATURES="${FEATURES} binpkg-signing '
'binpkg-request-signature"',
'BINPKG_FORMAT="gpkg"',
f'BINPKG_GPG_SIGNING_BASE_COMMAND="flock {gpg_test_path}/portage-binpkg-gpg.lock /usr/bin/gpg --sign --armor --batch --no-tty --yes --pinentry-mode loopback --passphrase GentooTest [PORTAGE_CONFIG]"',
'BINPKG_GPG_SIGNING_DIGEST="SHA512"',
f'BINPKG_GPG_SIGNING_GPG_HOME="{gpg_test_path}"',
'BINPKG_GPG_SIGNING_KEY="0x8812797DDF1DD192"',
'BINPKG_GPG_VERIFY_BASE_COMMAND="/usr/bin/gpg --verify --batch --no-tty --yes --no-auto-check-trustdb --status-fd 1 [PORTAGE_CONFIG] [SIGNATURE]"',
f'BINPKG_GPG_VERIFY_GPG_HOME="{gpg_test_path}"',
),
})
tmpdir = tempfile.mkdtemp()
try:
settings = playground.settings
gpg = GPG(settings)
gpg.unlock()
orig_full_path = os.path.join(tmpdir, "orig/")
os.makedirs(orig_full_path)
data = urandom(1048576)
with open(os.path.join(orig_full_path, "data"), "wb") as f:
f.write(data)
binpkg_1 = gpkg(settings, "test",
os.path.join(tmpdir, "test-1.gpkg.tar"))
binpkg_1.compress(orig_full_path, {})
binpkg_2 = gpkg(settings, "test",
os.path.join(tmpdir, "test-1.gpkg.tar"))
self.assertRaises(InvalidSignature, binpkg_2.decompress,
os.path.join(tmpdir, "test"))
finally:
shutil.rmtree(tmpdir)
playground.cleanup()
def test_gpkg_different_size_file(self):
if sys.version_info.major < 3:
self.skipTest("Not support Python 2")
playground = ResolverPlayground(
user_config={
"make.conf": (
'FEATURES="${FEATURES} -binpkg-signing '
'-binpkg-request-signature -gpg-keepalive"',
),
}
)
tmpdir = tempfile.mkdtemp()
try:
settings = playground.settings
orig_full_path = os.path.join(tmpdir, "orig/")
os.makedirs(orig_full_path)
data = urandom(100)
with open(os.path.join(orig_full_path, "data"), "wb") as f:
f.write(data)
binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
binpkg_1.compress(orig_full_path, {})
with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
with tarfile.open(
os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
) as tar_2:
for f in tar_1.getmembers():
tar_2.addfile(f, tar_1.extractfile(f))
tar_2.addfile(f, tar_1.extractfile(f))
binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
self.assertRaises(
InvalidBinaryPackageFormat,
binpkg_2.decompress,
os.path.join(tmpdir, "test"),
)
finally:
shutil.rmtree(tmpdir)
playground.cleanup()
def unpackinfo(self, mydest):
"""Unpacks all the files from the dataSegment into 'mydest'."""
if not self.scan():
return 0
try:
origdir = os.getcwd()
except SystemExit as e:
raise
except:
os.chdir("/")
origdir = "/"
a = open(
_unicode_encode(self.file,
encoding=_encodings['fs'],
errors='strict'), 'rb')
if not os.path.exists(mydest):
os.makedirs(mydest)
os.chdir(mydest)
startpos = 0
while ((startpos + 8) < self.indexsize):
namelen = decodeint(self.index[startpos:startpos + 4])
datapos = decodeint(self.index[startpos + 4 + namelen:startpos +
8 + namelen])
datalen = decodeint(self.index[startpos + 8 + namelen:startpos +
12 + namelen])
myname = self.index[startpos + 4:startpos + 4 + namelen]
myname = _unicode_decode(myname,
encoding=_encodings['repo.content'],
errors='replace')
dirname = os.path.dirname(myname)
if dirname:
if not os.path.exists(dirname):
os.makedirs(dirname)
mydat = open(
_unicode_encode(myname,
encoding=_encodings['fs'],
errors='strict'), 'wb')
a.seek(self.datapos + datapos)
mydat.write(a.read(datalen))
mydat.close()
startpos = startpos + namelen + 12
a.close()
os.chdir(origdir)
return 1
def _create_ebuilds(self, ebuilds):
for cpv in ebuilds:
a = Atom("=" + cpv)
ebuild_dir = os.path.join(self.portdir, a.cp)
ebuild_path = os.path.join(ebuild_dir,
a.cpv.split("/")[1] + ".ebuild")
try:
os.makedirs(ebuild_dir)
except os.error:
pass
metadata = ebuilds[cpv].copy()
eapi = metadata.pop("EAPI", 0)
lic = metadata.pop("LICENSE", 0)
properties = metadata.pop("PROPERTIES", "")
slot = metadata.pop("SLOT", 0)
keywords = metadata.pop("KEYWORDS", "x86")
iuse = metadata.pop("IUSE", "")
depend = metadata.pop("DEPEND", "")
rdepend = metadata.pop("RDEPEND", None)
pdepend = metadata.pop("PDEPEND", None)
required_use = metadata.pop("REQUIRED_USE", None)
if metadata:
raise ValueError(
"metadata of ebuild '%s' contains unknown keys: %s" %
(cpv, metadata.keys()))
f = open(ebuild_path, "w")
f.write('EAPI="' + str(eapi) + '"\n')
f.write('LICENSE="' + str(lic) + '"\n')
f.write('PROPERTIES="' + str(properties) + '"\n')
f.write('SLOT="' + str(slot) + '"\n')
f.write('KEYWORDS="' + str(keywords) + '"\n')
f.write('IUSE="' + str(iuse) + '"\n')
f.write('DEPEND="' + str(depend) + '"\n')
if rdepend is not None:
f.write('RDEPEND="' + str(rdepend) + '"\n')
if pdepend is not None:
f.write('PDEPEND="' + str(pdepend) + '"\n')
if required_use is not None:
f.write('REQUIRED_USE="' + str(required_use) + '"\n')
f.close()
def __init__(self, ebuilds={}, binpkgs={}, installed={}, profile={}, repo_configs={}, \
user_config={}, sets={}, world=[], world_sets=[], distfiles={},
targetroot=False, debug=False):
"""
ebuilds: cpv -> metadata mapping simulating available ebuilds.
installed: cpv -> metadata mapping simulating installed packages.
If a metadata key is missing, it gets a default value.
profile: settings defined by the profile.
"""
self.debug = debug
self.eprefix = normalize_path(tempfile.mkdtemp())
portage.const.EPREFIX = self.eprefix.rstrip(os.sep)
self.eroot = self.eprefix + os.sep
if targetroot:
self.target_root = os.path.join(self.eroot, 'target_root')
else:
self.target_root = os.sep
self.distdir = os.path.join(self.eroot, "var", "portage", "distfiles")
self.pkgdir = os.path.join(self.eprefix, "pkgdir")
self.vdbdir = os.path.join(self.eroot, "var/db/pkg")
os.makedirs(self.vdbdir)
if not debug:
portage.util.noiselimit = -2
self._repositories = {}
#Make sure the main repo is always created
self._get_repo_dir("test_repo")
self._create_distfiles(distfiles)
self._create_ebuilds(ebuilds)
self._create_binpkgs(binpkgs)
self._create_installed(installed)
self._create_profile(ebuilds, installed, profile, repo_configs,
user_config, sets)
self._create_world(world, world_sets)
self.settings, self.trees = self._load_config()
self._create_ebuild_manifests(ebuilds)
portage.util.noiselimit = 0
def test_gpkg_update_metadata(self):
if sys.version_info.major < 3:
self.skipTest("Not support Python 2")
playground = ResolverPlayground(
user_config={
"make.conf": ('BINPKG_COMPRESS="gzip"',),
}
)
tmpdir = tempfile.mkdtemp()
try:
settings = playground.settings
orig_full_path = os.path.join(tmpdir, "orig/")
os.makedirs(orig_full_path)
with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
test_file.write(urandom(1048576))
gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
test_gpkg = gpkg(settings, "test", gpkg_file_loc)
meta = {"test1": b"1234567890", "test2": b"abcdef"}
test_gpkg.compress(os.path.join(tmpdir, "orig"), meta)
meta_result = test_gpkg.get_metadata()
self.assertEqual(meta, meta_result)
meta_new = {"test3": b"0987654321", "test4": b"XXXXXXXX"}
test_gpkg.update_metadata(meta_new)
meta_result = test_gpkg.get_metadata()
self.assertEqual(meta_new, meta_result)
test_gpkg.decompress(os.path.join(tmpdir, "test"))
r = compare_files(
os.path.join(tmpdir, "orig/" + "test"),
os.path.join(tmpdir, "test/" + "test"),
skipped_types=("atime", "mtime", "ctime"),
)
self.assertEqual(r, ())
finally:
shutil.rmtree(tmpdir)
playground.cleanup()
def unpackinfo(self, mydest):
"""Unpacks all the files from the dataSegment into 'mydest'."""
if not self.scan():
return 0
mydest = normalize_path(mydest) + os.sep
a = open(
_unicode_encode(self.file, encoding=_encodings["fs"], errors="strict"), "rb"
)
if not os.path.exists(mydest):
os.makedirs(mydest)
startpos = 0
while (startpos + 8) < self.indexsize:
namelen = decodeint(self.index[startpos : startpos + 4])
datapos = decodeint(
self.index[startpos + 4 + namelen : startpos + 8 + namelen]
)
datalen = decodeint(
self.index[startpos + 8 + namelen : startpos + 12 + namelen]
)
myname = self.index[startpos + 4 : startpos + 4 + namelen]
myname = _unicode_decode(
myname, encoding=_encodings["repo.content"], errors="replace"
)
filename = os.path.join(mydest, myname.lstrip(os.sep))
filename = normalize_path(filename)
if not filename.startswith(mydest):
# myname contains invalid ../ component(s)
continue
dirname = os.path.dirname(filename)
if dirname:
if not os.path.exists(dirname):
os.makedirs(dirname)
mydat = open(
_unicode_encode(filename, encoding=_encodings["fs"], errors="strict"),
"wb",
)
a.seek(self.datapos + datapos)
mydat.write(a.read(datalen))
mydat.close()
startpos = startpos + namelen + 12
a.close()
return 1
def testFakedbapi(self):
packages = (
(
"sys-apps/portage-2.1.10",
{"EAPI": "2", "IUSE": "ipc doc", "repository": "gentoo", "SLOT": "0", "USE": "ipc missing-iuse"},
),
("virtual/package-manager-0", {"EAPI": "0", "repository": "gentoo", "SLOT": "0"}),
)
match_tests = (
("sys-apps/portage:0[ipc]", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0[-ipc]", []),
("sys-apps/portage:0[doc]", []),
("sys-apps/portage:0[-doc]", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0[missing-iuse]", []),
("sys-apps/portage:0[-missing-iuse]", []),
("sys-apps/portage:0::gentoo[ipc]", ["sys-apps/portage-2.1.10"]),
("sys-apps/portage:0::multilib[ipc]", []),
("virtual/package-manager", ["virtual/package-manager-0"]),
)
tempdir = tempfile.mkdtemp()
try:
test_repo = os.path.join(tempdir, "var", "repositories", "test_repo")
os.makedirs(os.path.join(test_repo, "profiles"))
with open(os.path.join(test_repo, "profiles", "repo_name"), "w") as f:
f.write("test_repo")
env = {"PORTAGE_REPOSITORIES": "[DEFAULT]\nmain-repo = test_repo\n[test_repo]\nlocation = %s" % test_repo}
fakedb = fakedbapi(settings=config(config_profile_path="", env=env, eprefix=tempdir))
for cpv, metadata in packages:
fakedb.cpv_inject(cpv, metadata=metadata)
for atom, expected_result in match_tests:
result = fakedb.match(atom)
self.assertEqual(
fakedb.match(atom),
expected_result,
"fakedb.match('%s') = %s != %s" % (atom, result, expected_result),
)
finally:
shutil.rmtree(tempdir)
def BuildTmp(self, tmp_subdir):
tmp_dir = mkdtemp()
hooks_dir = tmp_dir + "/" + tmp_subdir
os.makedirs(hooks_dir)
f = open(hooks_dir + "/1-testhook", "w")
f.write("#!/bin/bash\n")
f.write('export hookonlytest="portage cannot see me!"\n')
f.write("exit 0\n")
f.close()
f = open(hooks_dir + "/2-testhook", "w")
f.write("#!/bin/bash\n")
f.write(
'if [[ "${hookonlytest}" != "" ]]; then echo "Unexpected hookonlytest value: ${hookonlytest}"; exit 1; fi\n'
)
f.write("exit 0\n")
f.close()
return tmp_dir
def BuildTmp(self, tmp_subdir):
tmp_dir = mkdtemp()
hooks_dir = tmp_dir + '/' + tmp_subdir
os.makedirs(hooks_dir)
f = open(hooks_dir + '/1-testhook', 'w')
f.write('#!/bin/bash\n')
f.write('export hookonlytest="portage cannot see me!"\n')
f.write('exit 0\n')
f.close()
f = open(hooks_dir + '/2-testhook', 'w')
f.write('#!/bin/bash\n')
f.write(
'if [[ "${hookonlytest}" != "" ]]; then echo "Unexpected hookonlytest value: ${hookonlytest}"; exit 1; fi\n'
)
f.write('exit 0\n')
f.close()
return tmp_dir
def _create_ebuilds(self, ebuilds):
for cpv in ebuilds:
a = Atom("=" + cpv, allow_repo=True)
repo = a.repo
if repo is None:
repo = "test_repo"
metadata = ebuilds[cpv].copy()
copyright_header = metadata.pop("COPYRIGHT_HEADER", None)
eapi = metadata.pop("EAPI", "0")
misc_content = metadata.pop("MISC_CONTENT", None)
metadata.setdefault("DEPEND", "")
metadata.setdefault("SLOT", "0")
metadata.setdefault("KEYWORDS", "x86")
metadata.setdefault("IUSE", "")
unknown_keys = set(metadata).difference(
portage.dbapi.dbapi._known_keys)
if unknown_keys:
raise ValueError(
"metadata of ebuild '%s' contains unknown keys: %s" %
(cpv, sorted(unknown_keys)))
repo_dir = self._get_repo_dir(repo)
ebuild_dir = os.path.join(repo_dir, a.cp)
ebuild_path = os.path.join(ebuild_dir,
a.cpv.split("/")[1] + ".ebuild")
try:
os.makedirs(ebuild_dir)
except os.error:
pass
f = open(ebuild_path, "w")
if copyright_header is not None:
f.write(copyright_header)
f.write('EAPI="%s"\n' % eapi)
for k, v in metadata.items():
f.write('%s="%s"\n' % (k, v))
if misc_content is not None:
f.write(misc_content)
f.close()
def _create_ebuilds(self, ebuilds):
for cpv in ebuilds:
a = Atom("=" + cpv)
ebuild_dir = os.path.join(self.portdir, a.cp)
ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild")
try:
os.makedirs(ebuild_dir)
except os.error:
pass
metadata = ebuilds[cpv].copy()
eapi = metadata.pop("EAPI", 0)
lic = metadata.pop("LICENSE", 0)
properties = metadata.pop("PROPERTIES", "")
slot = metadata.pop("SLOT", 0)
keywords = metadata.pop("KEYWORDS", "x86")
iuse = metadata.pop("IUSE", "")
depend = metadata.pop("DEPEND", "")
rdepend = metadata.pop("RDEPEND", None)
pdepend = metadata.pop("PDEPEND", None)
required_use = metadata.pop("REQUIRED_USE", None)
if metadata:
raise ValueError("metadata of ebuild '%s' contains unknown keys: %s" % (cpv, metadata.keys()))
f = open(ebuild_path, "w")
f.write('EAPI="' + str(eapi) + '"\n')
f.write('LICENSE="' + str(lic) + '"\n')
f.write('PROPERTIES="' + str(properties) + '"\n')
f.write('SLOT="' + str(slot) + '"\n')
f.write('KEYWORDS="' + str(keywords) + '"\n')
f.write('IUSE="' + str(iuse) + '"\n')
f.write('DEPEND="' + str(depend) + '"\n')
if rdepend is not None:
f.write('RDEPEND="' + str(rdepend) + '"\n')
if pdepend is not None:
f.write('PDEPEND="' + str(pdepend) + '"\n')
if required_use is not None:
f.write('REQUIRED_USE="' + str(required_use) + '"\n')
f.close()
def ensure_dirs(dir_path, **kwargs):
"""Create a directory and call apply_permissions.
Returns True if a directory is created or the permissions needed to be
modified, and False otherwise.
This function's handling of EEXIST errors makes it useful for atomic
directory creation, in which multiple processes may be competing to
create the same directory.
"""
created_dir = False
try:
os.makedirs(dir_path)
created_dir = True
except OSError as oe:
func_call = "makedirs('%s')" % dir_path
if oe.errno in (errno.EEXIST,):
pass
else:
if os.path.isdir(dir_path):
# NOTE: DragonFly raises EPERM for makedir('/')
# and that is supposed to be ignored here.
# Also, sometimes mkdir raises EISDIR on FreeBSD
# and we want to ignore that too (bug #187518).
pass
elif oe.errno == errno.EPERM:
raise OperationNotPermitted(func_call)
elif oe.errno == errno.EACCES:
raise PermissionDenied(func_call)
elif oe.errno == errno.EROFS:
raise ReadOnlyFileSystem(func_call)
else:
raise
if kwargs:
perms_modified = apply_permissions(dir_path, **kwargs)
else:
perms_modified = False
return created_dir or perms_modified
def _get_repo_dir(self, repo):
"""
Create the repo directory if needed.
"""
if repo not in self._repositories:
if repo == "test_repo":
self._repositories["DEFAULT"] = {"main-repo": repo}
repo_path = os.path.join(self.eroot, "var", "repositories", repo)
self._repositories[repo] = {"location": repo_path}
profile_path = os.path.join(repo_path, "profiles")
try:
os.makedirs(profile_path)
except os.error:
pass
repo_name_file = os.path.join(profile_path, "repo_name")
with open(repo_name_file, "w") as f:
f.write("%s\n" % repo)
return self._repositories[repo]["location"]
def _get_repo_dir(self, repo):
"""
Create the repo directory if needed.
"""
if repo not in self._repositories:
if repo == "test_repo":
self._repositories["DEFAULT"] = {"main-repo": repo}
repo_path = os.path.join(self.eroot, "var", "repositories", repo)
self._repositories[repo] = {"location": repo_path}
profile_path = os.path.join(repo_path, "profiles")
try:
os.makedirs(profile_path)
except os.error:
pass
repo_name_file = os.path.join(profile_path, "repo_name")
with open(repo_name_file, "w") as f:
f.write("%s\n" % repo)
return self._repositories[repo]["location"]
def test_filename_hash_layout_get_filenames(self):
filename = DistfileName(
"foo-1.tar.gz",
digests=dict((algo, checksum_str(b"", hashname=algo))
for algo in MANIFEST2_HASH_DEFAULTS),
)
layouts = (
FlatLayout(),
FilenameHashLayout("SHA1", "4"),
FilenameHashLayout("SHA1", "8"),
FilenameHashLayout("SHA1", "8:16"),
FilenameHashLayout("SHA1", "8:16:24"),
ContentHashLayout("SHA512", "8:8:8"),
)
for layout in layouts:
distdir = tempfile.mkdtemp()
try:
path = os.path.join(distdir, layout.get_path(filename))
try:
os.makedirs(os.path.dirname(path))
except OSError:
pass
with open(path, "wb") as f:
pass
file_list = list(layout.get_filenames(distdir))
self.assertTrue(len(file_list) > 0)
for filename_result in file_list:
if isinstance(filename_result, DistfileName):
self.assertTrue(
filename_result.digests_equal(filename))
else:
self.assertEqual(filename_result, str(filename))
finally:
shutil.rmtree(distdir)
def _create_installed(self, installed):
for cpv in installed:
a = Atom("=" + cpv, allow_repo=True)
repo = a.repo
if repo is None:
repo = "test_repo"
vdb_pkg_dir = os.path.join(self.vdbdir, a.cpv)
try:
os.makedirs(vdb_pkg_dir)
except os.error:
pass
metadata = installed[cpv].copy()
metadata.setdefault("SLOT", "0")
metadata.setdefault("BUILD_TIME", "0")
metadata.setdefault("COUNTER", "0")
metadata.setdefault("KEYWORDS", "~x86")
unknown_keys = set(metadata).difference(
portage.dbapi.dbapi._known_keys)
unknown_keys.discard("BUILD_TIME")
unknown_keys.discard("BUILD_ID")
unknown_keys.discard("COUNTER")
unknown_keys.discard("repository")
unknown_keys.discard("USE")
unknown_keys.discard("PROVIDES")
unknown_keys.discard("REQUIRES")
if unknown_keys:
raise ValueError(
"metadata of installed '%s' contains unknown keys: %s" %
(cpv, sorted(unknown_keys)))
metadata["repository"] = repo
for k, v in metadata.items():
with open(os.path.join(vdb_pkg_dir, k), "w") as f:
f.write("%s\n" % v)
def __init__(self,
ebuilds={},
installed={},
profile={},
user_config={},
sets={},
world=[],
debug=False):
"""
ebuilds: cpv -> metadata mapping simulating avaiable ebuilds.
installed: cpv -> metadata mapping simulating installed packages.
If a metadata key is missing, it gets a default value.
profile: settings defined by the profile.
"""
self.debug = debug
self.root = "/"
self.eprefix = tempfile.mkdtemp()
self.eroot = self.root + self.eprefix.lstrip(os.sep) + os.sep
self.portdir = os.path.join(self.eroot, "usr/portage")
self.vdbdir = os.path.join(self.eroot, "var/db/pkg")
os.makedirs(self.portdir)
os.makedirs(self.vdbdir)
if not debug:
portage.util.noiselimit = -2
self._create_ebuilds(ebuilds)
self._create_installed(installed)
self._create_profile(ebuilds, installed, profile, user_config, sets)
self._create_world(world)
self.settings, self.trees = self._load_config()
self._create_ebuild_manifests(ebuilds)
portage.util.noiselimit = 0
def _get_repo_dir(self, repo):
"""
Create the repo directory if needed.
"""
if repo not in self.repo_dirs:
if repo == "test_repo":
repo_path = self.portdir
else:
repo_path = os.path.join(self.eroot, "usr", "local", repo)
self.repo_dirs[repo] = repo_path
profile_path = os.path.join(repo_path, "profiles")
try:
os.makedirs(profile_path)
except os.error:
pass
repo_name_file = os.path.join(profile_path, "repo_name")
f = open(repo_name_file, "w")
f.write("%s\n" % repo)
f.close()
return self.repo_dirs[repo]
def _create_profile(self, ebuilds, eclasses, installed, profile,
repo_configs, user_config, sets):
user_config_dir = os.path.join(self.eroot, USER_CONFIG_PATH)
try:
os.makedirs(user_config_dir)
except os.error:
pass
for repo in self._repositories:
if repo == "DEFAULT":
continue
repo_dir = self._get_repo_dir(repo)
profile_dir = os.path.join(repo_dir, "profiles")
metadata_dir = os.path.join(repo_dir, "metadata")
os.makedirs(metadata_dir)
# Create $REPO/profiles/categories
categories = set()
for cpv in ebuilds:
ebuilds_repo = Atom("=" + cpv, allow_repo=True).repo
if ebuilds_repo is None:
ebuilds_repo = "test_repo"
if ebuilds_repo == repo:
categories.add(catsplit(cpv)[0])
categories_file = os.path.join(profile_dir, "categories")
with open(categories_file, "w") as f:
for cat in categories:
f.write(cat + "\n")
# Create $REPO/profiles/license_groups
license_file = os.path.join(profile_dir, "license_groups")
with open(license_file, "w") as f:
f.write("EULA TEST\n")
repo_config = repo_configs.get(repo)
if repo_config:
for config_file, lines in repo_config.items():
if config_file not in self.config_files and not any(
fnmatch.fnmatch(config_file, os.path.join(x, "*"))
for x in self.config_files):
raise ValueError("Unknown config file: '%s'" %
config_file)
if config_file in ("layout.conf", ):
file_name = os.path.join(repo_dir, "metadata",
config_file)
else:
file_name = os.path.join(profile_dir, config_file)
if "/" in config_file and not os.path.isdir(
os.path.dirname(file_name)):
os.makedirs(os.path.dirname(file_name))
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
# Temporarily write empty value of masters until it becomes default.
# TODO: Delete all references to "# use implicit masters" when empty value becomes default.
if config_file == "layout.conf" and not any(
line.startswith(("masters =",
"# use implicit masters"))
for line in lines):
f.write("masters =\n")
# Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
eclass_dir = os.path.join(repo_dir, "eclass")
os.makedirs(eclass_dir)
for eclass_name, eclass_content in eclasses.items():
with open(
os.path.join(eclass_dir,
"{}.eclass".format(eclass_name)),
"wt") as f:
if isinstance(eclass_content, str):
eclass_content = [eclass_content]
for line in eclass_content:
f.write("{}\n".format(line))
# Temporarily write empty value of masters until it becomes default.
if not repo_config or "layout.conf" not in repo_config:
layout_conf_path = os.path.join(repo_dir, "metadata",
"layout.conf")
with open(layout_conf_path, "w") as f:
f.write("masters =\n")
if repo == "test_repo":
# Create a minimal profile in /var/db/repos/gentoo
sub_profile_dir = os.path.join(profile_dir, "default", "linux",
"x86", "test_profile")
os.makedirs(sub_profile_dir)
if not (profile and "eapi" in profile):
eapi_file = os.path.join(sub_profile_dir, "eapi")
with open(eapi_file, "w") as f:
f.write("0\n")
make_defaults_file = os.path.join(sub_profile_dir,
"make.defaults")
with open(make_defaults_file, "w") as f:
f.write('ARCH="x86"\n')
f.write('ACCEPT_KEYWORDS="x86"\n')
use_force_file = os.path.join(sub_profile_dir, "use.force")
with open(use_force_file, "w") as f:
f.write("x86\n")
parent_file = os.path.join(sub_profile_dir, "parent")
with open(parent_file, "w") as f:
f.write("..\n")
if profile:
for config_file, lines in profile.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" %
config_file)
file_name = os.path.join(sub_profile_dir, config_file)
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
# Create profile symlink
os.symlink(sub_profile_dir,
os.path.join(user_config_dir, "make.profile"))
gpg_test_path = os.environ["PORTAGE_GNUPGHOME"]
make_conf = {
"ACCEPT_KEYWORDS": "x86",
"BINPKG_GPG_SIGNING_BASE_COMMAND":
f"flock {gpg_test_path}/portage-binpkg-gpg.lock /usr/bin/gpg --sign --armor --yes --pinentry-mode loopback --passphrase GentooTest [PORTAGE_CONFIG]",
"BINPKG_GPG_SIGNING_GPG_HOME": gpg_test_path,
"BINPKG_GPG_SIGNING_KEY": "0x5D90EA06352177F6",
"BINPKG_GPG_VERIFY_GPG_HOME": gpg_test_path,
"CLEAN_DELAY": "0",
"DISTDIR": self.distdir,
"EMERGE_WARNING_DELAY": "0",
"FEATURES": "${FEATURES} binpkg-signing binpkg-request-signature "
"gpg-keepalive",
"PKGDIR": self.pkgdir,
"PORTAGE_INST_GID": str(portage.data.portage_gid),
"PORTAGE_INST_UID": str(portage.data.portage_uid),
"PORTAGE_TMPDIR": os.path.join(self.eroot, "var/tmp"),
}
if os.environ.get("NOCOLOR"):
make_conf["NOCOLOR"] = os.environ["NOCOLOR"]
# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
# need to be inherited by ebuild subprocesses.
if "PORTAGE_USERNAME" in os.environ:
make_conf["PORTAGE_USERNAME"] = os.environ["PORTAGE_USERNAME"]
if "PORTAGE_GRPNAME" in os.environ:
make_conf["PORTAGE_GRPNAME"] = os.environ["PORTAGE_GRPNAME"]
make_conf_lines = []
for k_v in make_conf.items():
make_conf_lines.append('%s="%s"' % k_v)
if "make.conf" in user_config:
make_conf_lines.extend(user_config["make.conf"])
if not portage.process.sandbox_capable or os.environ.get(
"SANDBOX_ON") == "1":
# avoid problems from nested sandbox instances
make_conf_lines.append(
'FEATURES="${FEATURES} -sandbox -usersandbox"')
configs = user_config.copy()
configs["make.conf"] = make_conf_lines
for config_file, lines in configs.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(user_config_dir, config_file)
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
# Create /usr/share/portage/config/make.globals
make_globals_path = os.path.join(self.eroot,
GLOBAL_CONFIG_PATH.lstrip(os.sep),
"make.globals")
ensure_dirs(os.path.dirname(make_globals_path))
os.symlink(os.path.join(cnf_path, "make.globals"), make_globals_path)
# Create /usr/share/portage/config/sets/portage.conf
default_sets_conf_dir = os.path.join(self.eroot,
"usr/share/portage/config/sets")
try:
os.makedirs(default_sets_conf_dir)
except os.error:
pass
provided_sets_portage_conf = os.path.join(cnf_path, "sets",
"portage.conf")
os.symlink(
provided_sets_portage_conf,
os.path.join(default_sets_conf_dir, "portage.conf"),
)
set_config_dir = os.path.join(user_config_dir, "sets")
try:
os.makedirs(set_config_dir)
except os.error:
pass
for sets_file, lines in sets.items():
file_name = os.path.join(set_config_dir, sets_file)
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
if cnf_path_repoman is not None:
# Create /usr/share/repoman
repoman_share_dir = os.path.join(self.eroot, "usr", "share",
"repoman")
os.symlink(cnf_path_repoman, repoman_share_dir)
def _create_distfiles(self, distfiles):
os.makedirs(self.distdir)
for k, v in distfiles.items():
with open(os.path.join(self.distdir, k), "wb") as f:
f.write(v)
def __init__(
self,
ebuilds={},
binpkgs={},
installed={},
profile={},
repo_configs={},
user_config={},
sets={},
world=[],
world_sets=[],
distfiles={},
eclasses={},
eprefix=None,
targetroot=False,
debug=False,
):
"""
ebuilds: cpv -> metadata mapping simulating available ebuilds.
installed: cpv -> metadata mapping simulating installed packages.
If a metadata key is missing, it gets a default value.
profile: settings defined by the profile.
"""
self.debug = debug
if eprefix is None:
self.eprefix = normalize_path(tempfile.mkdtemp())
# EPREFIX/bin is used by fake true_binaries. Real binaries goes into EPREFIX/usr/bin
eubin = os.path.join(self.eprefix, "usr", "bin")
ensure_dirs(eubin)
for x in self.portage_bin:
os.symlink(os.path.join(PORTAGE_BIN_PATH, x),
os.path.join(eubin, x))
eusbin = os.path.join(self.eprefix, "usr", "sbin")
ensure_dirs(eusbin)
for x in self.portage_sbin:
os.symlink(os.path.join(PORTAGE_BIN_PATH, x),
os.path.join(eusbin, x))
essential_binaries = (
"awk",
"basename",
"bzip2",
"cat",
"chgrp",
"chmod",
"chown",
"comm",
"cp",
"egrep",
"env",
"find",
"flock",
"grep",
"head",
"install",
"ln",
"mkdir",
"mkfifo",
"mktemp",
"mv",
"readlink",
"rm",
"sed",
"sort",
"tar",
"tr",
"uname",
"uniq",
"xargs",
"zstd",
)
# Exclude internal wrappers from PATH lookup.
orig_path = os.environ["PATH"]
included_paths = []
for path in orig_path.split(":"):
if path and not fnmatch.fnmatch(path,
"*/portage/*/ebuild-helpers*"):
included_paths.append(path)
try:
os.environ["PATH"] = ":".join(included_paths)
for x in essential_binaries:
path = find_binary(x)
if path is None:
raise portage.exception.CommandNotFound(x)
os.symlink(path, os.path.join(eubin, x))
finally:
os.environ["PATH"] = orig_path
else:
self.eprefix = normalize_path(eprefix)
# Tests may override portage.const.EPREFIX in order to
# simulate a prefix installation. It's reasonable to do
# this because tests should be self-contained such that
# the "real" value of portage.const.EPREFIX is entirely
# irrelevant (see bug #492932).
self._orig_eprefix = portage.const.EPREFIX
portage.const.EPREFIX = self.eprefix.rstrip(os.sep)
self.eroot = self.eprefix + os.sep
if targetroot:
self.target_root = os.path.join(self.eroot, "target_root")
else:
self.target_root = os.sep
self.distdir = os.path.join(self.eroot, "var", "portage", "distfiles")
self.pkgdir = os.path.join(self.eprefix, "pkgdir")
self.vdbdir = os.path.join(self.eroot, "var/db/pkg")
os.makedirs(self.vdbdir)
if not debug:
portage.util.noiselimit = -2
self._repositories = {}
# Make sure the main repo is always created
self._get_repo_dir("test_repo")
self._create_distfiles(distfiles)
self._create_ebuilds(ebuilds)
self._create_installed(installed)
self._create_profile(ebuilds, eclasses, installed, profile,
repo_configs, user_config, sets)
self._create_world(world, world_sets)
self.settings, self.trees = self._load_config()
self.gpg = None
self._create_binpkgs(binpkgs)
self._create_ebuild_manifests(ebuilds)
portage.util.noiselimit = 0
async def _async_test_simple(
self,
playground,
metadata_xml_files,
profiles,
profile,
licenses,
arch_list,
use_desc,
metadata_xsd,
copyright_header,
debug,
):
settings = playground.settings
eprefix = settings["EPREFIX"]
eroot = settings["EROOT"]
portdb = playground.trees[playground.eroot]["porttree"].dbapi
homedir = os.path.join(eroot, "home")
distdir = os.path.join(eprefix, "distdir")
test_repo_location = settings.repositories["test_repo"].location
profiles_dir = os.path.join(test_repo_location, "profiles")
license_dir = os.path.join(test_repo_location, "licenses")
repoman_cmd = (portage._python_interpreter, "-b", "-Wd",
os.path.join(self.bindir, "repoman"))
git_binary = find_binary("git")
git_cmd = (git_binary, )
cp_binary = find_binary("cp")
self.assertEqual(cp_binary is None, False, "cp command not found")
cp_cmd = (cp_binary, )
test_ebuild = portdb.findname("dev-libs/A-1")
self.assertFalse(test_ebuild is None)
committer_name = "Gentoo Dev"
committer_email = "*****@*****.**"
expected_warnings = {
"returncode": 0,
"warns": {
"variable.phase": [
"dev-libs/C/C-0.ebuild: line 15: phase pkg_preinst: EAPI 7: variable A: Forbidden reference to variable specified by PMS",
"dev-libs/C/C-0.ebuild: line 15: phase pkg_preinst: EAPI 7: variable BROOT: Forbidden reference to variable specified by PMS",
]
},
}
git_test = (
("", RepomanRun(args=["--version"])),
("", RepomanRun(args=["manifest"])),
("", git_cmd + (
"config",
"--global",
"user.name",
committer_name,
)),
("", git_cmd + (
"config",
"--global",
"user.email",
committer_email,
)),
("", git_cmd + ("init-db", )),
("", git_cmd + ("add", ".")),
("", git_cmd + ("commit", "-a", "-m", "add whole repo")),
("", RepomanRun(args=["full", "-d"], expected=expected_warnings)),
("",
RepomanRun(args=[
"full", "--include-profiles", "default/linux/x86/test_profile"
],
expected=expected_warnings)),
("", cp_cmd + (test_ebuild, test_ebuild[:-8] + "2.ebuild")),
("", git_cmd + ("add", test_ebuild[:-8] + "2.ebuild")),
("",
RepomanRun(args=["commit", "-m", "cat/pkg: bump to version 2"],
expected=expected_warnings)),
("", cp_cmd + (test_ebuild, test_ebuild[:-8] + "3.ebuild")),
("", git_cmd + ("add", test_ebuild[:-8] + "3.ebuild")),
("dev-libs",
RepomanRun(args=["commit", "-m", "cat/pkg: bump to version 3"],
expected=expected_warnings)),
("", cp_cmd + (test_ebuild, test_ebuild[:-8] + "4.ebuild")),
("", git_cmd + ("add", test_ebuild[:-8] + "4.ebuild")),
("dev-libs/A",
RepomanRun(args=["commit", "-m", "cat/pkg: bump to version 4"])),
)
env = {
"PORTAGE_OVERRIDE_EPREFIX":
eprefix,
"DISTDIR":
distdir,
"GENTOO_COMMITTER_NAME":
committer_name,
"GENTOO_COMMITTER_EMAIL":
committer_email,
"HOME":
homedir,
"PATH":
os.environ["PATH"],
"PORTAGE_GRPNAME":
os.environ["PORTAGE_GRPNAME"],
"PORTAGE_USERNAME":
os.environ["PORTAGE_USERNAME"],
"PORTAGE_REPOSITORIES":
settings.repositories.config_string(),
"PYTHONDONTWRITEBYTECODE":
os.environ.get("PYTHONDONTWRITEBYTECODE", ""),
}
if os.environ.get("SANDBOX_ON") == "1":
# avoid problems from nested sandbox instances
env["FEATURES"] = "-sandbox -usersandbox"
dirs = [homedir, license_dir, profiles_dir, distdir]
try:
for d in dirs:
ensure_dirs(d)
with open(os.path.join(test_repo_location, "skel.ChangeLog"),
'w') as f:
f.write(copyright_header)
with open(os.path.join(profiles_dir, "profiles.desc"), 'w') as f:
for x in profiles:
f.write("%s %s %s\n" % x)
# ResolverPlayground only created the first profile,
# so create the remaining ones.
for x in profiles[1:]:
sub_profile_dir = os.path.join(profiles_dir, x[1])
ensure_dirs(sub_profile_dir)
for config_file, lines in profile.items():
file_name = os.path.join(sub_profile_dir, config_file)
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
for x in licenses:
open(os.path.join(license_dir, x), 'wb').close()
with open(os.path.join(profiles_dir, "arch.list"), 'w') as f:
for x in arch_list:
f.write("%s\n" % x)
with open(os.path.join(profiles_dir, "use.desc"), 'w') as f:
for k, v in use_desc:
f.write("%s - %s\n" % (k, v))
for cp, xml_data in metadata_xml_files:
with open(os.path.join(test_repo_location, cp, "metadata.xml"),
'w') as f:
f.write(playground.metadata_xml_template % xml_data)
# Use a symlink to test_repo, in order to trigger bugs
# involving canonical vs. non-canonical paths.
test_repo_symlink = os.path.join(eroot, "test_repo_symlink")
os.symlink(test_repo_location, test_repo_symlink)
metadata_xsd_dest = os.path.join(
test_repo_location, 'metadata/xml-schema/metadata.xsd')
os.makedirs(os.path.dirname(metadata_xsd_dest))
os.symlink(metadata_xsd, metadata_xsd_dest)
if debug:
# The subprocess inherits both stdout and stderr, for
# debugging purposes.
stdout = None
else:
# The subprocess inherits stderr so that any warnings
# triggered by python -Wd will be visible.
stdout = subprocess.PIPE
for cwd in ("", "dev-libs", "dev-libs/A", "dev-libs/B",
"dev-libs/C"):
abs_cwd = os.path.join(test_repo_symlink, cwd)
proc = await asyncio.create_subprocess_exec(*(repoman_cmd +
("full", )),
env=env,
stderr=None,
stdout=stdout,
cwd=abs_cwd)
if debug:
await proc.wait()
else:
output, _err = await proc.communicate()
await proc.wait()
if proc.returncode != os.EX_OK:
portage.writemsg(output)
self.assertEqual(os.EX_OK, proc.returncode,
"repoman failed in %s" % (cwd, ))
if git_binary is not None:
for cwd, cmd in git_test:
abs_cwd = os.path.join(test_repo_symlink, cwd)
if isinstance(cmd, RepomanRun):
cmd.cwd = abs_cwd
cmd.env = env
cmd.debug = debug
await cmd.run()
if cmd.result[
"result"] != cmd.expected and cmd.result.get(
"stdio"):
portage.writemsg(cmd.result["stdio"])
try:
self.assertEqual(cmd.result["result"],
cmd.expected)
except Exception:
print(cmd.result["result"],
file=sys.stderr,
flush=True)
raise
continue
proc = await asyncio.create_subprocess_exec(*cmd,
env=env,
stderr=None,
stdout=stdout,
cwd=abs_cwd)
if debug:
await proc.wait()
else:
output, _err = await proc.communicate()
await proc.wait()
if proc.returncode != os.EX_OK:
portage.writemsg(output)
self.assertEqual(
os.EX_OK,
proc.returncode,
"%s failed in %s" % (
cmd,
cwd,
),
)
finally:
playground.cleanup()
def _create_profile(self, ebuilds, installed, profile, repo_configs,
user_config, sets):
user_config_dir = os.path.join(self.eroot, USER_CONFIG_PATH)
try:
os.makedirs(user_config_dir)
except os.error:
pass
for repo in self.repo_dirs:
repo_dir = self._get_repo_dir(repo)
profile_dir = os.path.join(self._get_repo_dir(repo), "profiles")
metadata_dir = os.path.join(repo_dir, "metadata")
os.makedirs(metadata_dir)
#Create $REPO/profiles/categories
categories = set()
for cpv in ebuilds:
ebuilds_repo = Atom("=" + cpv, allow_repo=True).repo
if ebuilds_repo is None:
ebuilds_repo = "test_repo"
if ebuilds_repo == repo:
categories.add(catsplit(cpv)[0])
categories_file = os.path.join(profile_dir, "categories")
f = open(categories_file, "w")
for cat in categories:
f.write(cat + "\n")
f.close()
#Create $REPO/profiles/license_groups
license_file = os.path.join(profile_dir, "license_groups")
f = open(license_file, "w")
f.write("EULA TEST\n")
f.close()
repo_config = repo_configs.get(repo)
if repo_config:
for config_file, lines in repo_config.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" %
config_file)
if config_file in ("layout.conf", ):
file_name = os.path.join(repo_dir, "metadata",
config_file)
else:
file_name = os.path.join(profile_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
#Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
os.makedirs(os.path.join(repo_dir, "eclass"))
if repo == "test_repo":
#Create a minimal profile in /usr/portage
sub_profile_dir = os.path.join(profile_dir, "default", "linux",
"x86", "test_profile")
os.makedirs(sub_profile_dir)
eapi_file = os.path.join(sub_profile_dir, "eapi")
f = open(eapi_file, "w")
f.write("0\n")
f.close()
make_defaults_file = os.path.join(sub_profile_dir,
"make.defaults")
f = open(make_defaults_file, "w")
f.write("ARCH=\"x86\"\n")
f.write("ACCEPT_KEYWORDS=\"x86\"\n")
f.close()
use_force_file = os.path.join(sub_profile_dir, "use.force")
f = open(use_force_file, "w")
f.write("x86\n")
f.close()
parent_file = os.path.join(sub_profile_dir, "parent")
f = open(parent_file, "w")
f.write("..\n")
f.close()
if profile:
for config_file, lines in profile.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" %
config_file)
file_name = os.path.join(sub_profile_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
#Create profile symlink
os.symlink(sub_profile_dir,
os.path.join(user_config_dir, "make.profile"))
#Create minimal herds.xml
herds_xml = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE herds SYSTEM "http://www.gentoo.org/dtd/herds.dtd">
<?xml-stylesheet href="/xsl/herds.xsl" type="text/xsl" ?>
<?xml-stylesheet href="/xsl/guide.xsl" type="text/xsl" ?>
<herds>
<herd>
<name>base-system</name>
<email>[email protected]</email>
<description>Core system utilities and libraries.</description>
<maintainer>
<email>[email protected]</email>
<name>Base System</name>
<role>Base System Maintainer</role>
</maintainer>
</herd>
</herds>
"""
with open(os.path.join(metadata_dir, "metadata.xml"),
'w') as f:
f.write(herds_xml)
# Write empty entries for each repository, in order to exercise
# RepoConfigLoader's repos.conf processing.
repos_conf_file = os.path.join(user_config_dir, "repos.conf")
f = open(repos_conf_file, "w")
for repo in sorted(self.repo_dirs.keys()):
f.write("[%s]\n" % repo)
f.write("\n")
f.close()
for config_file, lines in user_config.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(user_config_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
#Create /usr/share/portage/config/make.globals
make_globals_path = os.path.join(self.eroot,
GLOBAL_CONFIG_PATH.lstrip(os.sep),
"make.globals")
ensure_dirs(os.path.dirname(make_globals_path))
os.symlink(os.path.join(PORTAGE_BASE_PATH, "cnf", "make.globals"),
make_globals_path)
#Create /usr/share/portage/config/sets/portage.conf
default_sets_conf_dir = os.path.join(self.eroot,
"usr/share/portage/config/sets")
try:
os.makedirs(default_sets_conf_dir)
except os.error:
pass
provided_sets_portage_conf = \
os.path.join(PORTAGE_BASE_PATH, "cnf/sets/portage.conf")
os.symlink(provided_sets_portage_conf,
os.path.join(default_sets_conf_dir, "portage.conf"))
set_config_dir = os.path.join(user_config_dir, "sets")
try:
os.makedirs(set_config_dir)
except os.error:
pass
for sets_file, lines in sets.items():
file_name = os.path.join(set_config_dir, sets_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
user_config_dir = os.path.join(self.eroot, "etc", "portage")
try:
os.makedirs(user_config_dir)
except os.error:
pass
for config_file, lines in user_config.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(user_config_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
