def _create_binpkgs(self, binpkgs):
# When using BUILD_ID, there can be mutiple instances for the
# same cpv. Therefore, binpkgs may be an iterable instead of
# a dict.
items = getattr(binpkgs, 'items', None)
items = items() if items is not None else binpkgs
for cpv, metadata in items:
a = Atom("=" + cpv, allow_repo=True)
repo = a.repo
if repo is None:
repo = "test_repo"
pn = catsplit(a.cp)[1]
cat, pf = catsplit(a.cpv)
metadata = metadata.copy()
metadata.setdefault("SLOT", "0")
metadata.setdefault("KEYWORDS", "x86")
metadata.setdefault("BUILD_TIME", "0")
metadata["repository"] = repo
metadata["CATEGORY"] = cat
metadata["PF"] = pf
repo_dir = self.pkgdir
category_dir = os.path.join(repo_dir, cat)
if "BUILD_ID" in metadata:
binpkg_path = os.path.join(category_dir, pn,
"%s-%s.xpak"% (pf, metadata["BUILD_ID"]))
else:
binpkg_path = os.path.join(category_dir, pf + ".tbz2")
ensure_dirs(os.path.dirname(binpkg_path))
t = portage.xpak.tbz2(binpkg_path)
t.recompose_mem(portage.xpak.xpak_mem(metadata))
def _create_binpkgs(self, binpkgs):
# When using BUILD_ID, there can be mutiple instances for the
# same cpv. Therefore, binpkgs may be an iterable instead of
# a dict.
items = getattr(binpkgs, 'items', None)
items = items() if items is not None else binpkgs
for cpv, metadata in items:
a = Atom("=" + cpv, allow_repo=True)
repo = a.repo
if repo is None:
repo = "test_repo"
pn = catsplit(a.cp)[1]
cat, pf = catsplit(a.cpv)
metadata = metadata.copy()
metadata.setdefault("SLOT", "0")
metadata.setdefault("KEYWORDS", "x86")
metadata.setdefault("BUILD_TIME", "0")
metadata["repository"] = repo
metadata["CATEGORY"] = cat
metadata["PF"] = pf
repo_dir = self.pkgdir
category_dir = os.path.join(repo_dir, cat)
if "BUILD_ID" in metadata:
binpkg_path = os.path.join(
category_dir, pn,
"%s-%s.xpak" % (pf, metadata["BUILD_ID"]))
else:
binpkg_path = os.path.join(category_dir, pf + ".tbz2")
ensure_dirs(os.path.dirname(binpkg_path))
t = portage.xpak.tbz2(binpkg_path)
t.recompose_mem(portage.xpak.xpak_mem(metadata))
def _create_binpkgs(self, binpkgs):
# When using BUILD_ID, there can be mutiple instances for the
# same cpv. Therefore, binpkgs may be an iterable instead of
# a dict.
items = getattr(binpkgs, "items", None)
items = items() if items is not None else binpkgs
binpkg_format = self.settings.get("BINPKG_FORMAT",
SUPPORTED_GENTOO_BINPKG_FORMATS[0])
if binpkg_format == "gpkg":
if self.gpg is None:
self.gpg = GPG(self.settings)
self.gpg.unlock()
for cpv, metadata in items:
a = Atom("=" + cpv, allow_repo=True)
repo = a.repo
if repo is None:
repo = "test_repo"
pn = catsplit(a.cp)[1]
cat, pf = catsplit(a.cpv)
metadata = metadata.copy()
metadata.setdefault("SLOT", "0")
metadata.setdefault("KEYWORDS", "x86")
metadata.setdefault("BUILD_TIME", "0")
metadata["repository"] = repo
metadata["CATEGORY"] = cat
metadata["PF"] = pf
metadata["BINPKG_FORMAT"] = binpkg_format
repo_dir = self.pkgdir
category_dir = os.path.join(repo_dir, cat)
if "BUILD_ID" in metadata:
if binpkg_format == "xpak":
binpkg_path = os.path.join(
category_dir, pn,
"%s-%s.xpak" % (pf, metadata["BUILD_ID"]))
elif binpkg_format == "gpkg":
binpkg_path = os.path.join(
category_dir, pn,
"%s-%s.gpkg.tar" % (pf, metadata["BUILD_ID"]))
else:
raise InvalidBinaryPackageFormat(binpkg_format)
else:
if binpkg_format == "xpak":
binpkg_path = os.path.join(category_dir, pf + ".tbz2")
elif binpkg_format == "gpkg":
binpkg_path = os.path.join(category_dir, pf + ".gpkg.tar")
else:
raise InvalidBinaryPackageFormat(binpkg_format)
ensure_dirs(os.path.dirname(binpkg_path))
if binpkg_format == "xpak":
t = portage.xpak.tbz2(binpkg_path)
t.recompose_mem(portage.xpak.xpak_mem(metadata))
elif binpkg_format == "gpkg":
t = portage.gpkg.gpkg(self.settings, a.cpv, binpkg_path)
t.compress(os.path.dirname(binpkg_path), metadata)
else:
raise InvalidBinaryPackageFormat(binpkg_format)
def similar_name_search(dbs, atom):
cp_lower = atom.cp.lower()
cat, pkg = catsplit(cp_lower)
if cat == "null":
cat = None
all_cp = set()
for db in dbs:
all_cp.update(db.cp_all())
# discard dir containing no ebuilds
all_cp.discard(atom.cp)
orig_cp_map = {}
for cp_orig in all_cp:
orig_cp_map.setdefault(cp_orig.lower(), []).append(cp_orig)
all_cp = set(orig_cp_map)
if cat:
matches = difflib.get_close_matches(cp_lower, all_cp)
else:
pkg_to_cp = {}
for other_cp in list(all_cp):
other_pkg = catsplit(other_cp)[1]
if other_pkg == pkg:
# Check for non-identical package that
# differs only by upper/lower case.
identical = True
for cp_orig in orig_cp_map[other_cp]:
if catsplit(cp_orig)[1] != \
catsplit(atom.cp)[1]:
identical = False
break
if identical:
# discard dir containing no ebuilds
all_cp.discard(other_cp)
continue
pkg_to_cp.setdefault(other_pkg, set()).add(other_cp)
pkg_matches = difflib.get_close_matches(pkg, pkg_to_cp)
matches = []
for pkg_match in pkg_matches:
matches.extend(pkg_to_cp[pkg_match])
matches_orig_case = []
for cp in matches:
matches_orig_case.extend(orig_cp_map[cp])
return matches_orig_case
def similar_name_search(dbs, atom):
cp_lower = atom.cp.lower()
cat, pkg = catsplit(cp_lower)
if cat == "null":
cat = None
all_cp = set()
for db in dbs:
all_cp.update(db.cp_all())
# discard dir containing no ebuilds
all_cp.discard(atom.cp)
orig_cp_map = {}
for cp_orig in all_cp:
orig_cp_map.setdefault(cp_orig.lower(), []).append(cp_orig)
all_cp = set(orig_cp_map)
if cat:
matches = difflib.get_close_matches(cp_lower, all_cp)
else:
pkg_to_cp = {}
for other_cp in list(all_cp):
other_pkg = catsplit(other_cp)[1]
if other_pkg == pkg:
# Check for non-identical package that
# differs only by upper/lower case.
identical = True
for cp_orig in orig_cp_map[other_cp]:
if catsplit(cp_orig)[1] != \
catsplit(atom.cp)[1]:
identical = False
break
if identical:
# discard dir containing no ebuilds
all_cp.discard(other_cp)
continue
pkg_to_cp.setdefault(other_pkg, set()).add(other_cp)
pkg_matches = difflib.get_close_matches(pkg, pkg_to_cp)
matches = []
for pkg_match in pkg_matches:
matches.extend(pkg_to_cp[pkg_match])
matches_orig_case = []
for cp in matches:
matches_orig_case.extend(orig_cp_map[cp])
return matches_orig_case
def dep_expand(mydep, mydb=None, use_cache=1, settings=None):
'''
@rtype: Atom
'''
if not len(mydep):
return mydep
if mydep[0]=="*":
mydep=mydep[1:]
orig_dep = mydep
if isinstance(orig_dep, Atom):
mydep = orig_dep.cp
else:
mydep = orig_dep
has_cat = '/' in orig_dep
if not has_cat:
alphanum = re.search(r'\w', orig_dep)
if alphanum:
mydep = orig_dep[:alphanum.start()] + "null/" + \
orig_dep[alphanum.start():]
try:
mydep = Atom(mydep)
except InvalidAtom:
# Missing '=' prefix is allowed for backward compatibility.
if not isvalidatom("=" + mydep):
raise
mydep = Atom('=' + mydep)
orig_dep = '=' + orig_dep
if not has_cat:
null_cat, pn = catsplit(mydep.cp)
mydep = pn
else:
mydep = mydep.cp
expanded = cpv_expand(mydep, mydb=mydb,
use_cache=use_cache, settings=settings)
return Atom(orig_dep.replace(mydep, expanded, 1))
def dep_expand(mydep, mydb=None, use_cache=1, settings=None):
'''
@rtype: Atom
'''
if not len(mydep):
return mydep
if mydep[0]=="*":
mydep=mydep[1:]
orig_dep = mydep
if isinstance(orig_dep, Atom):
mydep = orig_dep.cp
else:
mydep = orig_dep
has_cat = '/' in orig_dep
if not has_cat:
alphanum = re.search(r'\w', orig_dep)
if alphanum:
mydep = orig_dep[:alphanum.start()] + "null/" + \
orig_dep[alphanum.start():]
try:
mydep = Atom(mydep)
except InvalidAtom:
# Missing '=' prefix is allowed for backward compatibility.
if not isvalidatom("=" + mydep):
raise
mydep = Atom('=' + mydep)
orig_dep = '=' + orig_dep
if not has_cat:
null_cat, pn = catsplit(mydep.cp)
mydep = pn
else:
mydep = mydep.cp
expanded = cpv_expand(mydep, mydb=mydb,
use_cache=use_cache, settings=settings)
return Atom(orig_dep.replace(mydep, expanded, 1))
def load(self): myatoms = [] for cp in self._db.cp_all(): if catsplit(cp)[0] == self._category: if (not self._check) or len(self._db.match(cp)) > 0: myatoms.append(cp) self._setAtoms(myatoms)
def load(self): myatoms = [] for cp in self._db.cp_all(): if catsplit(cp)[0] == self._category: if (not self._check) or len(self._db.match(cp)) > 0: myatoms.append(cp) self._setAtoms(myatoms)
def __init__(self, repo, pkgcand):
cat = catsplit(pkgcand)[0]
self._atom = PortageHackedAtom(
str(repo._atom).replace('null/', '%s/' % cat), repo._atom.repo)
self._dbapi = repo._dbapi
self._path = repo._path
self._pkg_class = repo._pkg_class
self._prio = repo._prio
self._name = repo._name
def is_private_package_atom(atom, installed_from=None, debug=False):
cat = catsplit(get_cp(atom))[0]
if GeneratedPackages().is_generated_cat(cat):
if GeneratedPackages().is_private_cat(cat):
if debug:
print ' skipping "%s" (%s, %s)' % \
(atom, 'was generated', 'is currently blacklisted')
return True
else:
if installed_from and \
installed_from[0] != '' and \
not GeneratedPackages().is_dedicated_repo_name(
installed_from[0]):
if debug:
print ' removing %s source tree "%s" for atom "%s"' % \
('misleading', installed_from[0], atom)
installed_from[0] = ''
if installed_from != None:
"""
- We collect private packages iff they come from a non-private
overlay, because that means they were in there before and are
actually not private. An example would be media-sound/xmms.
- We collect packages from private overlays iff the package also
exists in a non-private overlay. An example would be that
you posted your ebuild to bugs.gentoo.org and somebody added
it to the tree in the meantime.
"""
if not Overlays().is_private_overlay_name(installed_from[0]):
return False
if not Overlays().is_private_package_atom(atom):
if installed_from and installed_from[0] != '':
if debug:
print ' removing %s source tree "%s" for atom "%s"' % \
('private', installed_from[0], atom)
installed_from[0] = ''
return False
if debug:
print ' skipping "%s" (%s, %s)' % \
(atom, 'was installed from private tree',
'is currently not in non-private tree')
return True
else:
not_in_public_trees = Overlays().is_private_package_atom(atom)
if debug and not_in_public_trees:
print ' skipping "%s" (not in public trees)' % atom
return not_in_public_trees
def is_private_package_atom(atom, installed_from=None, debug=False):
cat = catsplit(get_cp(atom))[0]
if GeneratedPackages().is_generated_cat(cat):
if GeneratedPackages().is_private_cat(cat):
if debug:
print ' skipping "%s" (%s, %s)' % \
(atom, 'was generated', 'is currently blacklisted')
return True
else:
if installed_from and \
installed_from[0] != '' and \
not GeneratedPackages().is_dedicated_repo_name(
installed_from[0]):
if debug:
print ' removing %s source tree "%s" for atom "%s"' % \
('misleading', installed_from[0], atom)
installed_from[0] = ''
if installed_from != None:
"""
- We collect private packages iff they come from a non-private
overlay, because that means they were in there before and are
actually not private. An example would be media-sound/xmms.
- We collect packages from private overlays iff the package also
exists in a non-private overlay. An example would be that
you posted your ebuild to bugs.gentoo.org and somebody added
it to the tree in the meantime.
"""
if not Overlays().is_private_overlay_name(installed_from[0]):
return False
if not Overlays().is_private_package_atom(atom):
if installed_from and installed_from[0] != '':
if debug:
print ' removing %s source tree "%s" for atom "%s"' % \
('private', installed_from[0], atom)
installed_from[0] = ''
return False
if debug:
print ' skipping "%s" (%s, %s)' % \
(atom, 'was installed from private tree',
'is currently not in non-private tree')
return True
else:
not_in_public_trees = Overlays().is_private_package_atom(atom)
if debug and not_in_public_trees:
print ' skipping "%s" (not in public trees)' % atom
return not_in_public_trees
def dep_expand(mydep, mydb=None, use_cache=1, settings=None):
'''
@rtype: Atom
'''
orig_dep = mydep
if isinstance(orig_dep, Atom):
has_cat = True
else:
if not mydep:
return mydep
if mydep[0] == "*":
mydep = mydep[1:]
orig_dep = mydep
has_cat = '/' in orig_dep.split(':')[0]
if not has_cat:
alphanum = re.search(r'\w', orig_dep)
if alphanum:
mydep = orig_dep[:alphanum.start()] + "null/" + \
orig_dep[alphanum.start():]
try:
mydep = Atom(mydep, allow_repo=True)
except InvalidAtom:
# Missing '=' prefix is allowed for backward compatibility.
if not isvalidatom("=" + mydep, allow_repo=True):
raise
mydep = Atom('=' + mydep, allow_repo=True)
orig_dep = '=' + orig_dep
if not has_cat:
null_cat, pn = catsplit(mydep.cp)
mydep = pn
if has_cat:
# Optimize most common cases to avoid calling cpv_expand.
if not mydep.cp.startswith("virtual/"):
return mydep
if not hasattr(mydb, "cp_list") or \
mydb.cp_list(mydep.cp):
return mydep
# Fallback to legacy cpv_expand for old-style PROVIDE virtuals.
mydep = mydep.cp
expanded = cpv_expand(mydep,
mydb=mydb,
use_cache=use_cache,
settings=settings)
return Atom(orig_dep.replace(mydep, expanded, 1), allow_repo=True)
def dep_expand(mydep, mydb=None, use_cache=1, settings=None):
'''
@rtype: Atom
'''
orig_dep = mydep
if isinstance(orig_dep, Atom):
has_cat = True
else:
if not mydep:
return mydep
if mydep[0] == "*":
mydep = mydep[1:]
orig_dep = mydep
has_cat = '/' in orig_dep.split(':')[0]
if not has_cat:
alphanum = re.search(r'\w', orig_dep)
if alphanum:
mydep = orig_dep[:alphanum.start()] + "null/" + \
orig_dep[alphanum.start():]
try:
mydep = Atom(mydep, allow_repo=True)
except InvalidAtom:
# Missing '=' prefix is allowed for backward compatibility.
if not isvalidatom("=" + mydep, allow_repo=True):
raise
mydep = Atom('=' + mydep, allow_repo=True)
orig_dep = '=' + orig_dep
if not has_cat:
null_cat, pn = catsplit(mydep.cp)
mydep = pn
if has_cat:
# Optimize most common cases to avoid calling cpv_expand.
if not mydep.cp.startswith("virtual/"):
return mydep
if not hasattr(mydb, "cp_list") or \
mydb.cp_list(mydep.cp):
return mydep
# Fallback to legacy cpv_expand for old-style PROVIDE virtuals.
mydep = mydep.cp
expanded = cpv_expand(mydep, mydb=mydb,
use_cache=use_cache, settings=settings)
return Atom(orig_dep.replace(mydep, expanded, 1), allow_repo=True)
def _create_binpkgs(self, binpkgs):
for cpv, metadata in binpkgs.items():
a = Atom("=" + cpv, allow_repo=True)
repo = a.repo
if repo is None:
repo = "test_repo"
cat, pf = catsplit(a.cpv)
metadata = metadata.copy()
metadata.setdefault("SLOT", "0")
metadata.setdefault("KEYWORDS", "x86")
metadata.setdefault("BUILD_TIME", "0")
metadata["repository"] = repo
metadata["CATEGORY"] = cat
metadata["PF"] = pf
repo_dir = self.pkgdir
category_dir = os.path.join(repo_dir, cat)
binpkg_path = os.path.join(category_dir, pf + ".tbz2")
ensure_dirs(category_dir)
t = portage.xpak.tbz2(binpkg_path)
t.recompose_mem(portage.xpak.xpak_mem(metadata))
def _create_binpkgs(self, binpkgs):
for cpv, metadata in binpkgs.items():
a = Atom("=" + cpv, allow_repo=True)
repo = a.repo
if repo is None:
repo = "test_repo"
cat, pf = catsplit(a.cpv)
metadata = metadata.copy()
metadata.setdefault("SLOT", "0")
metadata.setdefault("KEYWORDS", "x86")
metadata.setdefault("BUILD_TIME", "0")
metadata["repository"] = repo
metadata["CATEGORY"] = cat
metadata["PF"] = pf
repo_dir = self.pkgdir
category_dir = os.path.join(repo_dir, cat)
binpkg_path = os.path.join(category_dir, pf + ".tbz2")
ensure_dirs(category_dir)
t = portage.xpak.tbz2(binpkg_path)
t.recompose_mem(portage.xpak.xpak_mem(metadata))
def digestgen(myarchives=None, mysettings=None,
overwrite=None, manifestonly=None, myportdb=None):
"""
Generates a digest file if missing. Fetches files if necessary.
NOTE: myarchives and mysettings used to be positional arguments,
so their order must be preserved for backward compatibility.
@param mysettings: the ebuild config (mysettings["O"] must correspond
to the ebuild's parent directory)
@type mysettings: config
@param myportdb: a portdbapi instance
@type myportdb: portdbapi
@rtype: int
@returns: 1 on success and 0 on failure
"""
if mysettings is None:
raise TypeError("portage.digestgen(): missing" + \
" required 'mysettings' parameter")
if myportdb is None:
warnings.warn("portage.digestgen() called without 'myportdb' parameter",
DeprecationWarning, stacklevel=2)
myportdb = portage.portdb
if overwrite is not None:
warnings.warn("portage.digestgen() called with " + \
"deprecated 'overwrite' parameter",
DeprecationWarning, stacklevel=2)
if manifestonly is not None:
warnings.warn("portage.digestgen() called with " + \
"deprecated 'manifestonly' parameter",
DeprecationWarning, stacklevel=2)
try:
portage._doebuild_manifest_exempt_depend += 1
distfiles_map = {}
fetchlist_dict = FetchlistDict(mysettings["O"], mysettings, myportdb)
for cpv in fetchlist_dict:
try:
for myfile in fetchlist_dict[cpv]:
distfiles_map.setdefault(myfile, []).append(cpv)
except InvalidDependString as e:
writemsg("!!! %s\n" % str(e), noiselevel=-1)
del e
return 0
mytree = os.path.dirname(os.path.dirname(mysettings["O"]))
manifest1_compat = False
mf = Manifest(mysettings["O"], mysettings["DISTDIR"],
fetchlist_dict=fetchlist_dict, manifest1_compat=manifest1_compat)
# Don't require all hashes since that can trigger excessive
# fetches when sufficient digests already exist. To ease transition
# while Manifest 1 is being removed, only require hashes that will
# exist before and after the transition.
required_hash_types = set()
required_hash_types.add("size")
required_hash_types.add(MANIFEST2_REQUIRED_HASH)
dist_hashes = mf.fhashdict.get("DIST", {})
# To avoid accidental regeneration of digests with the incorrect
# files (such as partially downloaded files), trigger the fetch
# code if the file exists and it's size doesn't match the current
# manifest entry. If there really is a legitimate reason for the
# digest to change, `ebuild --force digest` can be used to avoid
# triggering this code (or else the old digests can be manually
# removed from the Manifest).
missing_files = []
for myfile in distfiles_map:
myhashes = dist_hashes.get(myfile)
if not myhashes:
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError:
st = None
if st is None or st.st_size == 0:
missing_files.append(myfile)
continue
size = myhashes.get("size")
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError as e:
if e.errno != errno.ENOENT:
raise
del e
if size == 0:
missing_files.append(myfile)
continue
if required_hash_types.difference(myhashes):
missing_files.append(myfile)
continue
else:
if st.st_size == 0 or size is not None and size != st.st_size:
missing_files.append(myfile)
continue
if missing_files:
mytree = os.path.realpath(os.path.dirname(
os.path.dirname(mysettings["O"])))
fetch_settings = config(clone=mysettings)
debug = mysettings.get("PORTAGE_DEBUG") == "1"
for myfile in missing_files:
uris = set()
for cpv in distfiles_map[myfile]:
myebuild = os.path.join(mysettings["O"],
catsplit(cpv)[1] + ".ebuild")
# for RESTRICT=fetch, mirror, etc...
doebuild_environment(myebuild, "fetch",
mysettings["ROOT"], fetch_settings,
debug, 1, myportdb)
uris.update(myportdb.getFetchMap(
cpv, mytree=mytree)[myfile])
fetch_settings["A"] = myfile # for use by pkg_nofetch()
try:
st = os.stat(os.path.join(
mysettings["DISTDIR"],myfile))
except OSError:
st = None
if not fetch({myfile : uris}, fetch_settings):
writemsg(_("!!! Fetch failed for %s, can't update "
"Manifest\n") % myfile, noiselevel=-1)
if myfile in dist_hashes and \
st is not None and st.st_size > 0:
# stat result is obtained before calling fetch(),
# since fetch may rename the existing file if the
# digest does not match.
writemsg(_("!!! If you would like to "
"forcefully replace the existing "
"Manifest entry\n!!! for %s, use "
"the following command:\n") % myfile + \
"!!! " + colorize("INFORM",
"ebuild --force %s manifest" % \
os.path.basename(myebuild)) + "\n",
noiselevel=-1)
return 0
writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"])
try:
mf.create(assumeDistHashesSometimes=True,
assumeDistHashesAlways=(
"assume-digests" in mysettings.features))
except FileNotFound as e:
writemsg(_("!!! File %s doesn't exist, can't update "
"Manifest\n") % e, noiselevel=-1)
return 0
except PortagePackageException as e:
writemsg(("!!! %s\n") % (e,), noiselevel=-1)
return 0
try:
mf.write(sign=False)
except PermissionDenied as e:
writemsg(_("!!! Permission Denied: %s\n") % (e,), noiselevel=-1)
return 0
if "assume-digests" not in mysettings.features:
distlist = list(mf.fhashdict.get("DIST", {}))
distlist.sort()
auto_assumed = []
for filename in distlist:
if not os.path.exists(
os.path.join(mysettings["DISTDIR"], filename)):
auto_assumed.append(filename)
if auto_assumed:
mytree = os.path.realpath(
os.path.dirname(os.path.dirname(mysettings["O"])))
cp = os.path.sep.join(mysettings["O"].split(os.path.sep)[-2:])
pkgs = myportdb.cp_list(cp, mytree=mytree)
pkgs.sort()
writemsg_stdout(" digest.assumed" + colorize("WARN",
str(len(auto_assumed)).rjust(18)) + "\n")
for pkg_key in pkgs:
fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree)
pv = pkg_key.split("/")[1]
for filename in auto_assumed:
if filename in fetchlist:
writemsg_stdout(
" %s::%s\n" % (pv, filename))
return 1
finally:
portage._doebuild_manifest_exempt_depend -= 1
def _create_profile(self, ebuilds, installed, profile, repo_configs, user_config, sets):
user_config_dir = os.path.join(self.eroot, USER_CONFIG_PATH)
try:
os.makedirs(user_config_dir)
except os.error:
pass
for repo in self.repo_dirs:
repo_dir = self._get_repo_dir(repo)
profile_dir = os.path.join(self._get_repo_dir(repo), "profiles")
metadata_dir = os.path.join(repo_dir, "metadata")
os.makedirs(metadata_dir)
#Create $REPO/profiles/categories
categories = set()
for cpv in ebuilds:
ebuilds_repo = Atom("="+cpv, allow_repo=True).repo
if ebuilds_repo is None:
ebuilds_repo = "test_repo"
if ebuilds_repo == repo:
categories.add(catsplit(cpv)[0])
categories_file = os.path.join(profile_dir, "categories")
f = open(categories_file, "w")
for cat in categories:
f.write(cat + "\n")
f.close()
#Create $REPO/profiles/license_groups
license_file = os.path.join(profile_dir, "license_groups")
f = open(license_file, "w")
f.write("EULA TEST\n")
f.close()
repo_config = repo_configs.get(repo)
if repo_config:
for config_file, lines in repo_config.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
if config_file in ("layout.conf",):
file_name = os.path.join(repo_dir, "metadata", config_file)
else:
file_name = os.path.join(profile_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
#Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
os.makedirs(os.path.join(repo_dir, "eclass"))
if repo == "test_repo":
#Create a minimal profile in /usr/portage
sub_profile_dir = os.path.join(profile_dir, "default", "linux", "x86", "test_profile")
os.makedirs(sub_profile_dir)
eapi_file = os.path.join(sub_profile_dir, "eapi")
f = open(eapi_file, "w")
f.write("0\n")
f.close()
make_defaults_file = os.path.join(sub_profile_dir, "make.defaults")
f = open(make_defaults_file, "w")
f.write("ARCH=\"x86\"\n")
f.write("ACCEPT_KEYWORDS=\"x86\"\n")
f.close()
use_force_file = os.path.join(sub_profile_dir, "use.force")
f = open(use_force_file, "w")
f.write("x86\n")
f.close()
parent_file = os.path.join(sub_profile_dir, "parent")
f = open(parent_file, "w")
f.write("..\n")
f.close()
if profile:
for config_file, lines in profile.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(sub_profile_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
#Create profile symlink
os.symlink(sub_profile_dir, os.path.join(user_config_dir, "make.profile"))
#Create minimal herds.xml
herds_xml = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE herds SYSTEM "http://www.gentoo.org/dtd/herds.dtd">
<?xml-stylesheet href="/xsl/herds.xsl" type="text/xsl" ?>
<?xml-stylesheet href="/xsl/guide.xsl" type="text/xsl" ?>
<herds>
<herd>
<name>base-system</name>
<email>[email protected]</email>
<description>Core system utilities and libraries.</description>
<maintainer>
<email>[email protected]</email>
<name>Base System</name>
<role>Base System Maintainer</role>
</maintainer>
</herd>
</herds>
"""
with open(os.path.join(metadata_dir, "metadata.xml"), 'w') as f:
f.write(herds_xml)
# Write empty entries for each repository, in order to exercise
# RepoConfigLoader's repos.conf processing.
repos_conf_file = os.path.join(user_config_dir, "repos.conf")
f = open(repos_conf_file, "w")
for repo in sorted(self.repo_dirs.keys()):
f.write("[%s]\n" % repo)
f.write("\n")
f.close()
for config_file, lines in user_config.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(user_config_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
#Create /usr/share/portage/config/make.globals
make_globals_path = os.path.join(self.eroot,
GLOBAL_CONFIG_PATH.lstrip(os.sep), "make.globals")
ensure_dirs(os.path.dirname(make_globals_path))
os.symlink(os.path.join(PORTAGE_BASE_PATH, "cnf", "make.globals"),
make_globals_path)
#Create /usr/share/portage/config/sets/portage.conf
default_sets_conf_dir = os.path.join(self.eroot, "usr/share/portage/config/sets")
try:
os.makedirs(default_sets_conf_dir)
except os.error:
pass
provided_sets_portage_conf = \
os.path.join(PORTAGE_BASE_PATH, "cnf/sets/portage.conf")
os.symlink(provided_sets_portage_conf, os.path.join(default_sets_conf_dir, "portage.conf"))
set_config_dir = os.path.join(user_config_dir, "sets")
try:
os.makedirs(set_config_dir)
except os.error:
pass
for sets_file, lines in sets.items():
file_name = os.path.join(set_config_dir, sets_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
user_config_dir = os.path.join(self.eroot, "etc", "portage")
try:
os.makedirs(user_config_dir)
except os.error:
pass
for config_file, lines in user_config.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(user_config_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
def digestgen(myarchives=None, mysettings=None, myportdb=None):
"""
Generates a digest file if missing. Fetches files if necessary.
NOTE: myarchives and mysettings used to be positional arguments,
so their order must be preserved for backward compatibility.
@param mysettings: the ebuild config (mysettings["O"] must correspond
to the ebuild's parent directory)
@type mysettings: config
@param myportdb: a portdbapi instance
@type myportdb: portdbapi
@rtype: int
@return: 1 on success and 0 on failure
"""
if mysettings is None or myportdb is None:
raise TypeError(
"portage.digestgen(): 'mysettings' and 'myportdb' parameter are required."
)
try:
portage._doebuild_manifest_exempt_depend += 1
distfiles_map = {}
fetchlist_dict = FetchlistDict(mysettings["O"], mysettings, myportdb)
for cpv in fetchlist_dict:
try:
for myfile in fetchlist_dict[cpv]:
distfiles_map.setdefault(myfile, []).append(cpv)
except InvalidDependString as e:
writemsg("!!! %s\n" % str(e), noiselevel=-1)
del e
return 0
mytree = os.path.dirname(os.path.dirname(mysettings["O"]))
try:
mf = mysettings.repositories.get_repo_for_location(mytree)
except KeyError:
# backward compatibility
mytree = os.path.realpath(mytree)
mf = mysettings.repositories.get_repo_for_location(mytree)
repo_required_hashes = mf.manifest_required_hashes
if repo_required_hashes is None:
repo_required_hashes = MANIFEST2_HASH_DEFAULTS
mf = mf.load_manifest(mysettings["O"],
mysettings["DISTDIR"],
fetchlist_dict=fetchlist_dict)
if not mf.allow_create:
writemsg_stdout(
_(">>> Skipping creating Manifest for %s; "
"repository is configured to not use them\n") %
mysettings["O"])
return 1
# Don't require all hashes since that can trigger excessive
# fetches when sufficient digests already exist. To ease transition
# while Manifest 1 is being removed, only require hashes that will
# exist before and after the transition.
required_hash_types = set()
required_hash_types.add("size")
required_hash_types.update(repo_required_hashes)
dist_hashes = mf.fhashdict.get("DIST", {})
# To avoid accidental regeneration of digests with the incorrect
# files (such as partially downloaded files), trigger the fetch
# code if the file exists and it's size doesn't match the current
# manifest entry. If there really is a legitimate reason for the
# digest to change, `ebuild --force digest` can be used to avoid
# triggering this code (or else the old digests can be manually
# removed from the Manifest).
missing_files = []
for myfile in distfiles_map:
myhashes = dist_hashes.get(myfile)
if not myhashes:
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError:
st = None
if st is None or st.st_size == 0:
missing_files.append(myfile)
continue
size = myhashes.get("size")
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError as e:
if e.errno != errno.ENOENT:
raise
del e
if size == 0:
missing_files.append(myfile)
continue
if required_hash_types.difference(myhashes):
missing_files.append(myfile)
continue
else:
if st.st_size == 0 or size is not None and size != st.st_size:
missing_files.append(myfile)
continue
for myfile in missing_files:
uris = set()
all_restrict = set()
for cpv in distfiles_map[myfile]:
uris.update(myportdb.getFetchMap(cpv, mytree=mytree)[myfile])
restrict = myportdb.aux_get(cpv, ['RESTRICT'],
mytree=mytree)[0]
# Here we ignore conditional parts of RESTRICT since
# they don't apply unconditionally. Assume such
# conditionals only apply on the client side where
# digestgen() does not need to be called.
all_restrict.update(
use_reduce(restrict, flat=True, matchnone=True))
# fetch() uses CATEGORY and PF to display a message
# when fetch restriction is triggered.
cat, pf = catsplit(cpv)
mysettings["CATEGORY"] = cat
mysettings["PF"] = pf
# fetch() uses PORTAGE_RESTRICT to control fetch
# restriction, which is only applied to files that
# are not fetchable via a mirror:// URI.
mysettings["PORTAGE_RESTRICT"] = " ".join(all_restrict)
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError:
st = None
if not fetch({myfile: uris}, mysettings):
myebuild = os.path.join(mysettings["O"],
catsplit(cpv)[1] + ".ebuild")
spawn_nofetch(myportdb, myebuild)
writemsg(
_("!!! Fetch failed for %s, can't update Manifest\n") %
myfile,
noiselevel=-1)
if myfile in dist_hashes and \
st is not None and st.st_size > 0:
# stat result is obtained before calling fetch(),
# since fetch may rename the existing file if the
# digest does not match.
cmd = colorize(
"INFORM", "ebuild --force %s manifest" %
os.path.basename(myebuild))
writemsg((_(
"!!! If you would like to forcefully replace the existing Manifest entry\n"
"!!! for %s, use the following command:\n") % myfile) +
"!!! %s\n" % cmd,
noiselevel=-1)
return 0
writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"])
try:
mf.create(assumeDistHashesSometimes=True,
assumeDistHashesAlways=("assume-digests"
in mysettings.features))
except FileNotFound as e:
writemsg(_("!!! File %s doesn't exist, can't update Manifest\n") %
e,
noiselevel=-1)
return 0
except PortagePackageException as e:
writemsg(("!!! %s\n") % (e, ), noiselevel=-1)
return 0
try:
mf.write(sign=False)
except PermissionDenied as e:
writemsg(_("!!! Permission Denied: %s\n") % (e, ), noiselevel=-1)
return 0
if "assume-digests" not in mysettings.features:
distlist = list(mf.fhashdict.get("DIST", {}))
distlist.sort()
auto_assumed = []
for filename in distlist:
if not os.path.exists(
os.path.join(mysettings["DISTDIR"], filename)):
auto_assumed.append(filename)
if auto_assumed:
cp = os.path.sep.join(mysettings["O"].split(os.path.sep)[-2:])
pkgs = myportdb.cp_list(cp, mytree=mytree)
pkgs.sort()
writemsg_stdout(" digest.assumed" +
colorize("WARN",
str(len(auto_assumed)).rjust(18)) +
"\n")
for pkg_key in pkgs:
fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree)
pv = pkg_key.split("/")[1]
for filename in auto_assumed:
if filename in fetchlist:
writemsg_stdout(" %s::%s\n" % (pv, filename))
return 1
finally:
portage._doebuild_manifest_exempt_depend -= 1
def category(self):
return catsplit(self)[0]
def _create_profile(self, ebuilds, installed, profile, repo_configs,
user_config, sets):
user_config_dir = os.path.join(self.eroot, USER_CONFIG_PATH)
try:
os.makedirs(user_config_dir)
except os.error:
pass
for repo in self.repo_dirs:
repo_dir = self._get_repo_dir(repo)
profile_dir = os.path.join(self._get_repo_dir(repo), "profiles")
metadata_dir = os.path.join(repo_dir, "metadata")
os.makedirs(metadata_dir)
#Create $REPO/profiles/categories
categories = set()
for cpv in ebuilds:
ebuilds_repo = Atom("=" + cpv, allow_repo=True).repo
if ebuilds_repo is None:
ebuilds_repo = "test_repo"
if ebuilds_repo == repo:
categories.add(catsplit(cpv)[0])
categories_file = os.path.join(profile_dir, "categories")
f = open(categories_file, "w")
for cat in categories:
f.write(cat + "\n")
f.close()
#Create $REPO/profiles/license_groups
license_file = os.path.join(profile_dir, "license_groups")
f = open(license_file, "w")
f.write("EULA TEST\n")
f.close()
repo_config = repo_configs.get(repo)
if repo_config:
for config_file, lines in repo_config.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" %
config_file)
if config_file in ("layout.conf", ):
file_name = os.path.join(repo_dir, "metadata",
config_file)
else:
file_name = os.path.join(profile_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
#Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
os.makedirs(os.path.join(repo_dir, "eclass"))
if repo == "test_repo":
#Create a minimal profile in /usr/portage
sub_profile_dir = os.path.join(profile_dir, "default", "linux",
"x86", "test_profile")
os.makedirs(sub_profile_dir)
eapi_file = os.path.join(sub_profile_dir, "eapi")
f = open(eapi_file, "w")
f.write("0\n")
f.close()
make_defaults_file = os.path.join(sub_profile_dir,
"make.defaults")
f = open(make_defaults_file, "w")
f.write("ARCH=\"x86\"\n")
f.write("ACCEPT_KEYWORDS=\"x86\"\n")
f.close()
use_force_file = os.path.join(sub_profile_dir, "use.force")
f = open(use_force_file, "w")
f.write("x86\n")
f.close()
parent_file = os.path.join(sub_profile_dir, "parent")
f = open(parent_file, "w")
f.write("..\n")
f.close()
if profile:
for config_file, lines in profile.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" %
config_file)
file_name = os.path.join(sub_profile_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
#Create profile symlink
os.symlink(sub_profile_dir,
os.path.join(user_config_dir, "make.profile"))
#Create minimal herds.xml
herds_xml = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE herds SYSTEM "http://www.gentoo.org/dtd/herds.dtd">
<?xml-stylesheet href="/xsl/herds.xsl" type="text/xsl" ?>
<?xml-stylesheet href="/xsl/guide.xsl" type="text/xsl" ?>
<herds>
<herd>
<name>base-system</name>
<email>[email protected]</email>
<description>Core system utilities and libraries.</description>
<maintainer>
<email>[email protected]</email>
<name>Base System</name>
<role>Base System Maintainer</role>
</maintainer>
</herd>
</herds>
"""
with open(os.path.join(metadata_dir, "metadata.xml"),
'w') as f:
f.write(herds_xml)
# Write empty entries for each repository, in order to exercise
# RepoConfigLoader's repos.conf processing.
repos_conf_file = os.path.join(user_config_dir, "repos.conf")
f = open(repos_conf_file, "w")
for repo in sorted(self.repo_dirs.keys()):
f.write("[%s]\n" % repo)
f.write("\n")
f.close()
for config_file, lines in user_config.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(user_config_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
#Create /usr/share/portage/config/make.globals
make_globals_path = os.path.join(self.eroot,
GLOBAL_CONFIG_PATH.lstrip(os.sep),
"make.globals")
ensure_dirs(os.path.dirname(make_globals_path))
os.symlink(os.path.join(PORTAGE_BASE_PATH, "cnf", "make.globals"),
make_globals_path)
#Create /usr/share/portage/config/sets/portage.conf
default_sets_conf_dir = os.path.join(self.eroot,
"usr/share/portage/config/sets")
try:
os.makedirs(default_sets_conf_dir)
except os.error:
pass
provided_sets_portage_conf = \
os.path.join(PORTAGE_BASE_PATH, "cnf/sets/portage.conf")
os.symlink(provided_sets_portage_conf,
os.path.join(default_sets_conf_dir, "portage.conf"))
set_config_dir = os.path.join(user_config_dir, "sets")
try:
os.makedirs(set_config_dir)
except os.error:
pass
for sets_file, lines in sets.items():
file_name = os.path.join(set_config_dir, sets_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
user_config_dir = os.path.join(self.eroot, "etc", "portage")
try:
os.makedirs(user_config_dir)
except os.error:
pass
for config_file, lines in user_config.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(user_config_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
def __init__(self, repo, pkgcand):
cat = catsplit(pkgcand)[0]
self._atom = str(repo._atom).replace('null/', '%s/' % cat)
self._dbapi = repo._dbapi
self._pkg_class = repo._pkg_class
def _create_profile(self, ebuilds, installed, profile, user_config, sets):
#Create $PORTDIR/profiles/categories
categories = set()
for cpv in chain(ebuilds.keys(), installed.keys()):
categories.add(catsplit(cpv)[0])
profile_dir = os.path.join(self.portdir, "profiles")
try:
os.makedirs(profile_dir)
except os.error:
pass
categories_file = os.path.join(profile_dir, "categories")
f = open(categories_file, "w")
for cat in categories:
f.write(cat + "\n")
f.close()
#Create $REPO/profiles/license_groups
license_file = os.path.join(profile_dir, "license_groups")
f = open(license_file, "w")
f.write("EULA TEST\n")
f.close()
#Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
os.makedirs(os.path.join(self.portdir, "eclass"))
sub_profile_dir = os.path.join(profile_dir, "default", "linux", "x86",
"test_profile")
os.makedirs(sub_profile_dir)
eapi_file = os.path.join(sub_profile_dir, "eapi")
f = open(eapi_file, "w")
f.write("0\n")
f.close()
make_defaults_file = os.path.join(sub_profile_dir, "make.defaults")
f = open(make_defaults_file, "w")
f.write("ARCH=\"x86\"\n")
f.write("ACCEPT_KEYWORDS=\"x86\"\n")
f.close()
use_force_file = os.path.join(sub_profile_dir, "use.force")
f = open(use_force_file, "w")
f.write("x86\n")
f.close()
if profile:
#This is meant to allow the consumer to set up his own profile,
#with package.mask and what not.
raise NotImplementedError()
#Create profile symlink
os.makedirs(os.path.join(self.eroot, "etc"))
os.symlink(sub_profile_dir,
os.path.join(self.eroot, "etc", "make.profile"))
user_config_dir = os.path.join(self.eroot, "etc", "portage")
try:
os.makedirs(user_config_dir)
except os.error:
pass
for config_file, lines in user_config.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(user_config_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
#Create /usr/share/portage/config/sets/portage.conf
default_sets_conf_dir = os.path.join(self.eroot,
"usr/share/portage/config/sets")
try:
os.makedirs(default_sets_conf_dir)
except os.error:
pass
provided_sets_portage_conf = \
os.path.join(PORTAGE_BASE_PATH, "cnf/sets/portage.conf")
os.symlink(provided_sets_portage_conf,
os.path.join(default_sets_conf_dir, "portage.conf"))
set_config_dir = os.path.join(user_config_dir, "sets")
try:
os.makedirs(set_config_dir)
except os.error:
pass
for sets_file, lines in sets.items():
file_name = os.path.join(set_config_dir, sets_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
def _create_profile(self, ebuilds, eclasses, installed, profile,
repo_configs, user_config, sets):
user_config_dir = os.path.join(self.eroot, USER_CONFIG_PATH)
try:
os.makedirs(user_config_dir)
except os.error:
pass
for repo in self._repositories:
if repo == "DEFAULT":
continue
repo_dir = self._get_repo_dir(repo)
profile_dir = os.path.join(repo_dir, "profiles")
metadata_dir = os.path.join(repo_dir, "metadata")
os.makedirs(metadata_dir)
# Create $REPO/profiles/categories
categories = set()
for cpv in ebuilds:
ebuilds_repo = Atom("=" + cpv, allow_repo=True).repo
if ebuilds_repo is None:
ebuilds_repo = "test_repo"
if ebuilds_repo == repo:
categories.add(catsplit(cpv)[0])
categories_file = os.path.join(profile_dir, "categories")
with open(categories_file, "w") as f:
for cat in categories:
f.write(cat + "\n")
# Create $REPO/profiles/license_groups
license_file = os.path.join(profile_dir, "license_groups")
with open(license_file, "w") as f:
f.write("EULA TEST\n")
repo_config = repo_configs.get(repo)
if repo_config:
for config_file, lines in repo_config.items():
if config_file not in self.config_files and not any(
fnmatch.fnmatch(config_file, os.path.join(x, "*"))
for x in self.config_files):
raise ValueError("Unknown config file: '%s'" %
config_file)
if config_file in ("layout.conf", ):
file_name = os.path.join(repo_dir, "metadata",
config_file)
else:
file_name = os.path.join(profile_dir, config_file)
if "/" in config_file and not os.path.isdir(
os.path.dirname(file_name)):
os.makedirs(os.path.dirname(file_name))
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
# Temporarily write empty value of masters until it becomes default.
# TODO: Delete all references to "# use implicit masters" when empty value becomes default.
if config_file == "layout.conf" and not any(
line.startswith(("masters =",
"# use implicit masters"))
for line in lines):
f.write("masters =\n")
# Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
eclass_dir = os.path.join(repo_dir, "eclass")
os.makedirs(eclass_dir)
for eclass_name, eclass_content in eclasses.items():
with open(
os.path.join(eclass_dir,
"{}.eclass".format(eclass_name)),
"wt") as f:
if isinstance(eclass_content, str):
eclass_content = [eclass_content]
for line in eclass_content:
f.write("{}\n".format(line))
# Temporarily write empty value of masters until it becomes default.
if not repo_config or "layout.conf" not in repo_config:
layout_conf_path = os.path.join(repo_dir, "metadata",
"layout.conf")
with open(layout_conf_path, "w") as f:
f.write("masters =\n")
if repo == "test_repo":
# Create a minimal profile in /var/db/repos/gentoo
sub_profile_dir = os.path.join(profile_dir, "default", "linux",
"x86", "test_profile")
os.makedirs(sub_profile_dir)
if not (profile and "eapi" in profile):
eapi_file = os.path.join(sub_profile_dir, "eapi")
with open(eapi_file, "w") as f:
f.write("0\n")
make_defaults_file = os.path.join(sub_profile_dir,
"make.defaults")
with open(make_defaults_file, "w") as f:
f.write('ARCH="x86"\n')
f.write('ACCEPT_KEYWORDS="x86"\n')
use_force_file = os.path.join(sub_profile_dir, "use.force")
with open(use_force_file, "w") as f:
f.write("x86\n")
parent_file = os.path.join(sub_profile_dir, "parent")
with open(parent_file, "w") as f:
f.write("..\n")
if profile:
for config_file, lines in profile.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" %
config_file)
file_name = os.path.join(sub_profile_dir, config_file)
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
# Create profile symlink
os.symlink(sub_profile_dir,
os.path.join(user_config_dir, "make.profile"))
gpg_test_path = os.environ["PORTAGE_GNUPGHOME"]
make_conf = {
"ACCEPT_KEYWORDS": "x86",
"BINPKG_GPG_SIGNING_BASE_COMMAND":
f"flock {gpg_test_path}/portage-binpkg-gpg.lock /usr/bin/gpg --sign --armor --yes --pinentry-mode loopback --passphrase GentooTest [PORTAGE_CONFIG]",
"BINPKG_GPG_SIGNING_GPG_HOME": gpg_test_path,
"BINPKG_GPG_SIGNING_KEY": "0x5D90EA06352177F6",
"BINPKG_GPG_VERIFY_GPG_HOME": gpg_test_path,
"CLEAN_DELAY": "0",
"DISTDIR": self.distdir,
"EMERGE_WARNING_DELAY": "0",
"FEATURES": "${FEATURES} binpkg-signing binpkg-request-signature "
"gpg-keepalive",
"PKGDIR": self.pkgdir,
"PORTAGE_INST_GID": str(portage.data.portage_gid),
"PORTAGE_INST_UID": str(portage.data.portage_uid),
"PORTAGE_TMPDIR": os.path.join(self.eroot, "var/tmp"),
}
if os.environ.get("NOCOLOR"):
make_conf["NOCOLOR"] = os.environ["NOCOLOR"]
# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
# need to be inherited by ebuild subprocesses.
if "PORTAGE_USERNAME" in os.environ:
make_conf["PORTAGE_USERNAME"] = os.environ["PORTAGE_USERNAME"]
if "PORTAGE_GRPNAME" in os.environ:
make_conf["PORTAGE_GRPNAME"] = os.environ["PORTAGE_GRPNAME"]
make_conf_lines = []
for k_v in make_conf.items():
make_conf_lines.append('%s="%s"' % k_v)
if "make.conf" in user_config:
make_conf_lines.extend(user_config["make.conf"])
if not portage.process.sandbox_capable or os.environ.get(
"SANDBOX_ON") == "1":
# avoid problems from nested sandbox instances
make_conf_lines.append(
'FEATURES="${FEATURES} -sandbox -usersandbox"')
configs = user_config.copy()
configs["make.conf"] = make_conf_lines
for config_file, lines in configs.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(user_config_dir, config_file)
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
# Create /usr/share/portage/config/make.globals
make_globals_path = os.path.join(self.eroot,
GLOBAL_CONFIG_PATH.lstrip(os.sep),
"make.globals")
ensure_dirs(os.path.dirname(make_globals_path))
os.symlink(os.path.join(cnf_path, "make.globals"), make_globals_path)
# Create /usr/share/portage/config/sets/portage.conf
default_sets_conf_dir = os.path.join(self.eroot,
"usr/share/portage/config/sets")
try:
os.makedirs(default_sets_conf_dir)
except os.error:
pass
provided_sets_portage_conf = os.path.join(cnf_path, "sets",
"portage.conf")
os.symlink(
provided_sets_portage_conf,
os.path.join(default_sets_conf_dir, "portage.conf"),
)
set_config_dir = os.path.join(user_config_dir, "sets")
try:
os.makedirs(set_config_dir)
except os.error:
pass
for sets_file, lines in sets.items():
file_name = os.path.join(set_config_dir, sets_file)
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
if cnf_path_repoman is not None:
# Create /usr/share/repoman
repoman_share_dir = os.path.join(self.eroot, "usr", "share",
"repoman")
os.symlink(cnf_path_repoman, repoman_share_dir)
def _cpv(self):
cpv = self._pkg._cpv
return 'null/%s' % catsplit(cpv)[1]
def __get_path(self,cpv): cat,pn = catsplit(cpv_getkey(cpv)) return os.path.join(self.portdir,cat,pn,os.path.basename(cpv) + ".ebuild")
def __new__(self, s):
a = _get_atom(s)
if catsplit(a.cp)[0] == 'null':
return UnexpandedPortageAtom(a)
else:
return CompletePortageAtom(a)
def __get_path(self, cpv):
cat, pn = catsplit(cpv_getkey(cpv))
return os.path.join(self.portdir, cat, pn,
os.path.basename(cpv) + ".ebuild")
def match_from_list(mydep, candidate_list):
"""
Searches list for entries that matches the package.
@param mydep: The package atom to match
@type mydep: String
@param candidate_list: The list of package atoms to compare against
@param candidate_list: List
@rtype: List
@return: A list of package atoms that match the given package atom
"""
if not candidate_list:
return []
from portage.util import writemsg
if "!" == mydep[:1]:
mydep = mydep[1:]
if not isinstance(mydep, Atom):
mydep = Atom(mydep)
mycpv = mydep.cpv
mycpv_cps = catpkgsplit(mycpv) # Can be None if not specific
slot = mydep.slot
if not mycpv_cps:
cat, pkg = catsplit(mycpv)
ver = None
rev = None
else:
cat, pkg, ver, rev = mycpv_cps
if mydep == mycpv:
raise KeyError(_("Specific key requires an operator"
" (%s) (try adding an '=')") % (mydep))
if ver and rev:
operator = mydep.operator
if not operator:
writemsg(_("!!! Invalid atom: %s\n") % mydep, noiselevel=-1)
return []
else:
operator = None
mylist = []
if operator is None:
for x in candidate_list:
cp = getattr(x, "cp", None)
if cp is None:
mysplit = catpkgsplit(remove_slot(x))
if mysplit is not None:
cp = mysplit[0] + '/' + mysplit[1]
if cp != mycpv:
continue
mylist.append(x)
elif operator == "=": # Exact match
for x in candidate_list:
xcpv = getattr(x, "cpv", None)
if xcpv is None:
xcpv = remove_slot(x)
if not cpvequal(xcpv, mycpv):
continue
mylist.append(x)
elif operator == "=*": # glob match
# XXX: Nasty special casing for leading zeros
# Required as =* is a literal prefix match, so can't
# use vercmp
mysplit = catpkgsplit(mycpv)
myver = mysplit[2].lstrip("0")
if not myver or not myver[0].isdigit():
myver = "0"+myver
mycpv = mysplit[0]+"/"+mysplit[1]+"-"+myver
for x in candidate_list:
xs = getattr(x, "cpv_split", None)
if xs is None:
xs = catpkgsplit(remove_slot(x))
myver = xs[2].lstrip("0")
if not myver or not myver[0].isdigit():
myver = "0"+myver
xcpv = xs[0]+"/"+xs[1]+"-"+myver
if xcpv.startswith(mycpv):
mylist.append(x)
elif operator == "~": # version, any revision, match
for x in candidate_list:
xs = getattr(x, "cpv_split", None)
if xs is None:
xs = catpkgsplit(remove_slot(x))
if xs is None:
raise InvalidData(x)
if not cpvequal(xs[0]+"/"+xs[1]+"-"+xs[2], mycpv_cps[0]+"/"+mycpv_cps[1]+"-"+mycpv_cps[2]):
continue
if xs[2] != ver:
continue
mylist.append(x)
elif operator in [">", ">=", "<", "<="]:
mysplit = ["%s/%s" % (cat, pkg), ver, rev]
for x in candidate_list:
xs = getattr(x, "cpv_split", None)
if xs is None:
xs = catpkgsplit(remove_slot(x))
xcat, xpkg, xver, xrev = xs
xs = ["%s/%s" % (xcat, xpkg), xver, xrev]
try:
result = pkgcmp(xs, mysplit)
except ValueError: # pkgcmp may return ValueError during int() conversion
writemsg(_("\nInvalid package name: %s\n") % x, noiselevel=-1)
raise
if result is None:
continue
elif operator == ">":
if result > 0:
mylist.append(x)
elif operator == ">=":
if result >= 0:
mylist.append(x)
elif operator == "<":
if result < 0:
mylist.append(x)
elif operator == "<=":
if result <= 0:
mylist.append(x)
else:
raise KeyError(_("Unknown operator: %s") % mydep)
else:
raise KeyError(_("Unknown operator: %s") % mydep)
if slot is not None:
candidate_list = mylist
mylist = []
for x in candidate_list:
xslot = getattr(x, "slot", False)
if xslot is False:
xslot = dep_getslot(x)
if xslot is not None and xslot != slot:
continue
mylist.append(x)
if mydep.use:
candidate_list = mylist
mylist = []
for x in candidate_list:
use = getattr(x, "use", None)
if use is not None:
is_valid_flag = x.iuse.is_valid_flag
missing_iuse = False
for y in mydep.use.required:
if not is_valid_flag(y):
missing_iuse = True
break
if missing_iuse:
continue
if mydep.use.enabled.difference(use.enabled):
continue
if mydep.use.disabled.intersection(use.enabled):
continue
mylist.append(x)
return mylist
def digestgen(myarchives=None, mysettings=None, myportdb=None):
"""
Generates a digest file if missing. Fetches files if necessary.
NOTE: myarchives and mysettings used to be positional arguments,
so their order must be preserved for backward compatibility.
@param mysettings: the ebuild config (mysettings["O"] must correspond
to the ebuild's parent directory)
@type mysettings: config
@param myportdb: a portdbapi instance
@type myportdb: portdbapi
@rtype: int
@return: 1 on success and 0 on failure
"""
if mysettings is None or myportdb is None:
raise TypeError("portage.digestgen(): 'mysettings' and 'myportdb' parameter are required.")
try:
portage._doebuild_manifest_exempt_depend += 1
distfiles_map = {}
fetchlist_dict = FetchlistDict(mysettings["O"], mysettings, myportdb)
for cpv in fetchlist_dict:
try:
for myfile in fetchlist_dict[cpv]:
distfiles_map.setdefault(myfile, []).append(cpv)
except InvalidDependString as e:
writemsg("!!! %s\n" % str(e), noiselevel=-1)
del e
return 0
mytree = os.path.dirname(os.path.dirname(mysettings["O"]))
try:
mf = mysettings.repositories.get_repo_for_location(mytree)
except KeyError:
# backward compatibility
mytree = os.path.realpath(mytree)
mf = mysettings.repositories.get_repo_for_location(mytree)
mf = mf.load_manifest(mysettings["O"], mysettings["DISTDIR"], fetchlist_dict=fetchlist_dict)
if not mf.allow_create:
writemsg_stdout(
_(">>> Skipping creating Manifest for %s; " "repository is configured to not use them\n")
% mysettings["O"]
)
return 1
# Don't require all hashes since that can trigger excessive
# fetches when sufficient digests already exist. To ease transition
# while Manifest 1 is being removed, only require hashes that will
# exist before and after the transition.
required_hash_types = set()
required_hash_types.add("size")
required_hash_types.add(MANIFEST2_REQUIRED_HASH)
dist_hashes = mf.fhashdict.get("DIST", {})
# To avoid accidental regeneration of digests with the incorrect
# files (such as partially downloaded files), trigger the fetch
# code if the file exists and it's size doesn't match the current
# manifest entry. If there really is a legitimate reason for the
# digest to change, `ebuild --force digest` can be used to avoid
# triggering this code (or else the old digests can be manually
# removed from the Manifest).
missing_files = []
for myfile in distfiles_map:
myhashes = dist_hashes.get(myfile)
if not myhashes:
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError:
st = None
if st is None or st.st_size == 0:
missing_files.append(myfile)
continue
size = myhashes.get("size")
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError as e:
if e.errno != errno.ENOENT:
raise
del e
if size == 0:
missing_files.append(myfile)
continue
if required_hash_types.difference(myhashes):
missing_files.append(myfile)
continue
else:
if st.st_size == 0 or size is not None and size != st.st_size:
missing_files.append(myfile)
continue
if missing_files:
for myfile in missing_files:
uris = set()
all_restrict = set()
for cpv in distfiles_map[myfile]:
uris.update(myportdb.getFetchMap(cpv, mytree=mytree)[myfile])
restrict = myportdb.aux_get(cpv, ["RESTRICT"], mytree=mytree)[0]
# Here we ignore conditional parts of RESTRICT since
# they don't apply unconditionally. Assume such
# conditionals only apply on the client side where
# digestgen() does not need to be called.
all_restrict.update(use_reduce(restrict, flat=True, matchnone=True))
# fetch() uses CATEGORY and PF to display a message
# when fetch restriction is triggered.
cat, pf = catsplit(cpv)
mysettings["CATEGORY"] = cat
mysettings["PF"] = pf
# fetch() uses PORTAGE_RESTRICT to control fetch
# restriction, which is only applied to files that
# are not fetchable via a mirror:// URI.
mysettings["PORTAGE_RESTRICT"] = " ".join(all_restrict)
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError:
st = None
if not fetch({myfile: uris}, mysettings):
myebuild = os.path.join(mysettings["O"], catsplit(cpv)[1] + ".ebuild")
spawn_nofetch(myportdb, myebuild)
writemsg(_("!!! Fetch failed for %s, can't update " "Manifest\n") % myfile, noiselevel=-1)
if myfile in dist_hashes and st is not None and st.st_size > 0:
# stat result is obtained before calling fetch(),
# since fetch may rename the existing file if the
# digest does not match.
writemsg(
_(
"!!! If you would like to "
"forcefully replace the existing "
"Manifest entry\n!!! for %s, use "
"the following command:\n"
)
% myfile
+ "!!! "
+ colorize("INFORM", "ebuild --force %s manifest" % os.path.basename(myebuild))
+ "\n",
noiselevel=-1,
)
return 0
writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"])
try:
mf.create(assumeDistHashesSometimes=True, assumeDistHashesAlways=("assume-digests" in mysettings.features))
except FileNotFound as e:
writemsg(_("!!! File %s doesn't exist, can't update " "Manifest\n") % e, noiselevel=-1)
return 0
except PortagePackageException as e:
writemsg(("!!! %s\n") % (e,), noiselevel=-1)
return 0
try:
mf.write(sign=False)
except PermissionDenied as e:
writemsg(_("!!! Permission Denied: %s\n") % (e,), noiselevel=-1)
return 0
if "assume-digests" not in mysettings.features:
distlist = list(mf.fhashdict.get("DIST", {}))
distlist.sort()
auto_assumed = []
for filename in distlist:
if not os.path.exists(os.path.join(mysettings["DISTDIR"], filename)):
auto_assumed.append(filename)
if auto_assumed:
cp = os.path.sep.join(mysettings["O"].split(os.path.sep)[-2:])
pkgs = myportdb.cp_list(cp, mytree=mytree)
pkgs.sort()
writemsg_stdout(" digest.assumed" + colorize("WARN", str(len(auto_assumed)).rjust(18)) + "\n")
for pkg_key in pkgs:
fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree)
pv = pkg_key.split("/")[1]
for filename in auto_assumed:
if filename in fetchlist:
writemsg_stdout(" %s::%s\n" % (pv, filename))
return 1
finally:
portage._doebuild_manifest_exempt_depend -= 1
def _create_profile(self, ebuilds, installed, profile, repo_configs, user_config, sets):
user_config_dir = os.path.join(self.eroot, USER_CONFIG_PATH)
try:
os.makedirs(user_config_dir)
except os.error:
pass
for repo in self._repositories:
if repo == "DEFAULT":
continue
repo_dir = self._get_repo_dir(repo)
profile_dir = os.path.join(repo_dir, "profiles")
metadata_dir = os.path.join(repo_dir, "metadata")
os.makedirs(metadata_dir)
#Create $REPO/profiles/categories
categories = set()
for cpv in ebuilds:
ebuilds_repo = Atom("="+cpv, allow_repo=True).repo
if ebuilds_repo is None:
ebuilds_repo = "test_repo"
if ebuilds_repo == repo:
categories.add(catsplit(cpv)[0])
categories_file = os.path.join(profile_dir, "categories")
with open(categories_file, "w") as f:
for cat in categories:
f.write(cat + "\n")
#Create $REPO/profiles/license_groups
license_file = os.path.join(profile_dir, "license_groups")
with open(license_file, "w") as f:
f.write("EULA TEST\n")
repo_config = repo_configs.get(repo)
if repo_config:
for config_file, lines in repo_config.items():
if config_file not in self.config_files and not any(fnmatch.fnmatch(config_file, os.path.join(x, "*")) for x in self.config_files):
raise ValueError("Unknown config file: '%s'" % config_file)
if config_file in ("layout.conf",):
file_name = os.path.join(repo_dir, "metadata", config_file)
else:
file_name = os.path.join(profile_dir, config_file)
if "/" in config_file and not os.path.isdir(os.path.dirname(file_name)):
os.makedirs(os.path.dirname(file_name))
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
# Temporarily write empty value of masters until it becomes default.
# TODO: Delete all references to "# use implicit masters" when empty value becomes default.
if config_file == "layout.conf" and not any(line.startswith(("masters =", "# use implicit masters")) for line in lines):
f.write("masters =\n")
#Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
os.makedirs(os.path.join(repo_dir, "eclass"))
# Temporarily write empty value of masters until it becomes default.
if not repo_config or "layout.conf" not in repo_config:
layout_conf_path = os.path.join(repo_dir, "metadata", "layout.conf")
with open(layout_conf_path, "w") as f:
f.write("masters =\n")
if repo == "test_repo":
#Create a minimal profile in /usr/portage
sub_profile_dir = os.path.join(profile_dir, "default", "linux", "x86", "test_profile")
os.makedirs(sub_profile_dir)
if not (profile and "eapi" in profile):
eapi_file = os.path.join(sub_profile_dir, "eapi")
with open(eapi_file, "w") as f:
f.write("0\n")
make_defaults_file = os.path.join(sub_profile_dir, "make.defaults")
with open(make_defaults_file, "w") as f:
f.write("ARCH=\"x86\"\n")
f.write("ACCEPT_KEYWORDS=\"x86\"\n")
use_force_file = os.path.join(sub_profile_dir, "use.force")
with open(use_force_file, "w") as f:
f.write("x86\n")
parent_file = os.path.join(sub_profile_dir, "parent")
with open(parent_file, "w") as f:
f.write("..\n")
if profile:
for config_file, lines in profile.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(sub_profile_dir, config_file)
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
#Create profile symlink
os.symlink(sub_profile_dir, os.path.join(user_config_dir, "make.profile"))
make_conf = {
"ACCEPT_KEYWORDS": "x86",
"CLEAN_DELAY": "0",
"DISTDIR" : self.distdir,
"EMERGE_WARNING_DELAY": "0",
"PKGDIR": self.pkgdir,
"PORTAGE_INST_GID": str(portage.data.portage_gid),
"PORTAGE_INST_UID": str(portage.data.portage_uid),
"PORTAGE_TMPDIR": os.path.join(self.eroot, 'var/tmp'),
}
if os.environ.get("NOCOLOR"):
make_conf["NOCOLOR"] = os.environ["NOCOLOR"]
# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
# need to be inherited by ebuild subprocesses.
if 'PORTAGE_USERNAME' in os.environ:
make_conf['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
if 'PORTAGE_GRPNAME' in os.environ:
make_conf['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
make_conf_lines = []
for k_v in make_conf.items():
make_conf_lines.append('%s="%s"' % k_v)
if "make.conf" in user_config:
make_conf_lines.extend(user_config["make.conf"])
if not portage.process.sandbox_capable or \
os.environ.get("SANDBOX_ON") == "1":
# avoid problems from nested sandbox instances
make_conf_lines.append('FEATURES="${FEATURES} -sandbox -usersandbox"')
configs = user_config.copy()
configs["make.conf"] = make_conf_lines
for config_file, lines in configs.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(user_config_dir, config_file)
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
#Create /usr/share/portage/config/make.globals
make_globals_path = os.path.join(self.eroot,
GLOBAL_CONFIG_PATH.lstrip(os.sep), "make.globals")
ensure_dirs(os.path.dirname(make_globals_path))
os.symlink(os.path.join(cnf_path, "make.globals"),
make_globals_path)
#Create /usr/share/portage/config/sets/portage.conf
default_sets_conf_dir = os.path.join(self.eroot, "usr/share/portage/config/sets")
try:
os.makedirs(default_sets_conf_dir)
except os.error:
pass
provided_sets_portage_conf = (
os.path.join(cnf_path, "sets", "portage.conf"))
os.symlink(provided_sets_portage_conf, os.path.join(default_sets_conf_dir, "portage.conf"))
set_config_dir = os.path.join(user_config_dir, "sets")
try:
os.makedirs(set_config_dir)
except os.error:
pass
for sets_file, lines in sets.items():
file_name = os.path.join(set_config_dir, sets_file)
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
def _create_profile(self, ebuilds, installed, profile, repo_configs, user_config, sets):
user_config_dir = os.path.join(self.eroot, USER_CONFIG_PATH)
try:
os.makedirs(user_config_dir)
except os.error:
pass
for repo in self._repositories:
if repo == "DEFAULT":
continue
repo_dir = self._get_repo_dir(repo)
profile_dir = os.path.join(repo_dir, "profiles")
metadata_dir = os.path.join(repo_dir, "metadata")
os.makedirs(metadata_dir)
#Create $REPO/profiles/categories
categories = set()
for cpv in ebuilds:
ebuilds_repo = Atom("="+cpv, allow_repo=True).repo
if ebuilds_repo is None:
ebuilds_repo = "test_repo"
if ebuilds_repo == repo:
categories.add(catsplit(cpv)[0])
categories_file = os.path.join(profile_dir, "categories")
f = open(categories_file, "w")
for cat in categories:
f.write(cat + "\n")
f.close()
#Create $REPO/profiles/license_groups
license_file = os.path.join(profile_dir, "license_groups")
f = open(license_file, "w")
f.write("EULA TEST\n")
f.close()
repo_config = repo_configs.get(repo)
if repo_config:
for config_file, lines in repo_config.items():
if config_file not in self.config_files and not any(fnmatch.fnmatch(config_file, os.path.join(x, "*")) for x in self.config_files):
raise ValueError("Unknown config file: '%s'" % config_file)
if config_file in ("layout.conf",):
file_name = os.path.join(repo_dir, "metadata", config_file)
else:
file_name = os.path.join(profile_dir, config_file)
if "/" in config_file and not os.path.isdir(os.path.dirname(file_name)):
os.makedirs(os.path.dirname(file_name))
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
#Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
os.makedirs(os.path.join(repo_dir, "eclass"))
if repo == "test_repo":
#Create a minimal profile in /usr/portage
sub_profile_dir = os.path.join(profile_dir, "default", "linux", "x86", "test_profile")
os.makedirs(sub_profile_dir)
if not (profile and "eapi" in profile):
eapi_file = os.path.join(sub_profile_dir, "eapi")
f = open(eapi_file, "w")
f.write("0\n")
f.close()
make_defaults_file = os.path.join(sub_profile_dir, "make.defaults")
f = open(make_defaults_file, "w")
f.write("ARCH=\"x86\"\n")
f.write("ACCEPT_KEYWORDS=\"x86\"\n")
f.close()
use_force_file = os.path.join(sub_profile_dir, "use.force")
f = open(use_force_file, "w")
f.write("x86\n")
f.close()
parent_file = os.path.join(sub_profile_dir, "parent")
f = open(parent_file, "w")
f.write("..\n")
f.close()
if profile:
for config_file, lines in profile.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(sub_profile_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
#Create profile symlink
os.symlink(sub_profile_dir, os.path.join(user_config_dir, "make.profile"))
#Create minimal herds.xml
herds_xml = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE herds SYSTEM "http://www.gentoo.org/dtd/herds.dtd">
<?xml-stylesheet href="/xsl/herds.xsl" type="text/xsl" ?>
<?xml-stylesheet href="/xsl/guide.xsl" type="text/xsl" ?>
<herds>
<herd>
<name>base-system</name>
<email>[email protected]</email>
<description>Core system utilities and libraries.</description>
<maintainer>
<email>[email protected]</email>
<name>Base System</name>
<role>Base System Maintainer</role>
</maintainer>
</herd>
</herds>
"""
with open(os.path.join(metadata_dir, "metadata.xml"), 'w') as f:
f.write(herds_xml)
make_conf = {
"ACCEPT_KEYWORDS": "x86",
"CLEAN_DELAY": "0",
"DISTDIR" : self.distdir,
"EMERGE_WARNING_DELAY": "0",
"PKGDIR": self.pkgdir,
"PORTAGE_INST_GID": str(portage.data.portage_gid),
"PORTAGE_INST_UID": str(portage.data.portage_uid),
"PORTAGE_TMPDIR": os.path.join(self.eroot, 'var/tmp'),
}
if os.environ.get("NOCOLOR"):
make_conf["NOCOLOR"] = os.environ["NOCOLOR"]
# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
# need to be inherited by ebuild subprocesses.
if 'PORTAGE_USERNAME' in os.environ:
make_conf['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
if 'PORTAGE_GRPNAME' in os.environ:
make_conf['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
make_conf_lines = []
for k_v in make_conf.items():
make_conf_lines.append('%s="%s"' % k_v)
if "make.conf" in user_config:
make_conf_lines.extend(user_config["make.conf"])
if not portage.process.sandbox_capable or \
os.environ.get("SANDBOX_ON") == "1":
# avoid problems from nested sandbox instances
make_conf_lines.append('FEATURES="${FEATURES} -sandbox -usersandbox"')
configs = user_config.copy()
configs["make.conf"] = make_conf_lines
for config_file, lines in configs.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(user_config_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
#Create /usr/share/portage/config/make.globals
make_globals_path = os.path.join(self.eroot,
GLOBAL_CONFIG_PATH.lstrip(os.sep), "make.globals")
ensure_dirs(os.path.dirname(make_globals_path))
os.symlink(os.path.join(PORTAGE_BASE_PATH, "cnf", "make.globals"),
make_globals_path)
#Create /usr/share/portage/config/sets/portage.conf
default_sets_conf_dir = os.path.join(self.eroot, "usr/share/portage/config/sets")
try:
os.makedirs(default_sets_conf_dir)
except os.error:
pass
provided_sets_portage_conf = \
os.path.join(PORTAGE_BASE_PATH, "cnf/sets/portage.conf")
os.symlink(provided_sets_portage_conf, os.path.join(default_sets_conf_dir, "portage.conf"))
set_config_dir = os.path.join(user_config_dir, "sets")
try:
os.makedirs(set_config_dir)
except os.error:
pass
for sets_file, lines in sets.items():
file_name = os.path.join(set_config_dir, sets_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
def __new__(self, key):
return PMIncompletePackageKey.__new__(self, catsplit(key)[1])
def package(self):
return catsplit(self)[1]
def _create_profile(self, ebuilds, installed, profile, repo_configs,
user_config, sets):
user_config_dir = os.path.join(self.eroot, USER_CONFIG_PATH)
try:
os.makedirs(user_config_dir)
except os.error:
pass
for repo in self._repositories:
if repo == "DEFAULT":
continue
repo_dir = self._get_repo_dir(repo)
profile_dir = os.path.join(repo_dir, "profiles")
metadata_dir = os.path.join(repo_dir, "metadata")
os.makedirs(metadata_dir)
#Create $REPO/profiles/categories
categories = set()
for cpv in ebuilds:
ebuilds_repo = Atom("=" + cpv, allow_repo=True).repo
if ebuilds_repo is None:
ebuilds_repo = "test_repo"
if ebuilds_repo == repo:
categories.add(catsplit(cpv)[0])
categories_file = os.path.join(profile_dir, "categories")
with open(categories_file, "w") as f:
for cat in categories:
f.write(cat + "\n")
#Create $REPO/profiles/license_groups
license_file = os.path.join(profile_dir, "license_groups")
with open(license_file, "w") as f:
f.write("EULA TEST\n")
repo_config = repo_configs.get(repo)
if repo_config:
for config_file, lines in repo_config.items():
if config_file not in self.config_files and not any(
fnmatch.fnmatch(config_file, os.path.join(x, "*"))
for x in self.config_files):
raise ValueError("Unknown config file: '%s'" %
config_file)
if config_file in ("layout.conf", ):
file_name = os.path.join(repo_dir, "metadata",
config_file)
else:
file_name = os.path.join(profile_dir, config_file)
if "/" in config_file and not os.path.isdir(
os.path.dirname(file_name)):
os.makedirs(os.path.dirname(file_name))
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
# Temporarily write empty value of masters until it becomes default.
# TODO: Delete all references to "# use implicit masters" when empty value becomes default.
if config_file == "layout.conf" and not any(
line.startswith(("masters =",
"# use implicit masters"))
for line in lines):
f.write("masters =\n")
#Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
os.makedirs(os.path.join(repo_dir, "eclass"))
# Temporarily write empty value of masters until it becomes default.
if not repo_config or "layout.conf" not in repo_config:
layout_conf_path = os.path.join(repo_dir, "metadata",
"layout.conf")
with open(layout_conf_path, "w") as f:
f.write("masters =\n")
if repo == "test_repo":
#Create a minimal profile in /usr/portage
sub_profile_dir = os.path.join(profile_dir, "default", "linux",
"x86", "test_profile")
os.makedirs(sub_profile_dir)
if not (profile and "eapi" in profile):
eapi_file = os.path.join(sub_profile_dir, "eapi")
with open(eapi_file, "w") as f:
f.write("0\n")
make_defaults_file = os.path.join(sub_profile_dir,
"make.defaults")
with open(make_defaults_file, "w") as f:
f.write("ARCH=\"x86\"\n")
f.write("ACCEPT_KEYWORDS=\"x86\"\n")
use_force_file = os.path.join(sub_profile_dir, "use.force")
with open(use_force_file, "w") as f:
f.write("x86\n")
parent_file = os.path.join(sub_profile_dir, "parent")
with open(parent_file, "w") as f:
f.write("..\n")
if profile:
for config_file, lines in profile.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" %
config_file)
file_name = os.path.join(sub_profile_dir, config_file)
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
#Create profile symlink
os.symlink(sub_profile_dir,
os.path.join(user_config_dir, "make.profile"))
#Create minimal herds.xml
herds_xml = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE herds SYSTEM "http://www.gentoo.org/dtd/herds.dtd">
<?xml-stylesheet href="/xsl/herds.xsl" type="text/xsl" ?>
<?xml-stylesheet href="/xsl/guide.xsl" type="text/xsl" ?>
<herds>
<herd>
<name>base-system</name>
<email>[email protected]</email>
<description>Core system utilities and libraries.</description>
<maintainer>
<email>[email protected]</email>
<name>Base System</name>
<role>Base System Maintainer</role>
</maintainer>
</herd>
</herds>
"""
with open(os.path.join(metadata_dir, "metadata.xml"),
'w') as f:
f.write(herds_xml)
make_conf = {
"ACCEPT_KEYWORDS": "x86",
"CLEAN_DELAY": "0",
"DISTDIR": self.distdir,
"EMERGE_WARNING_DELAY": "0",
"PKGDIR": self.pkgdir,
"PORTAGE_INST_GID": str(portage.data.portage_gid),
"PORTAGE_INST_UID": str(portage.data.portage_uid),
"PORTAGE_TMPDIR": os.path.join(self.eroot, 'var/tmp'),
}
if os.environ.get("NOCOLOR"):
make_conf["NOCOLOR"] = os.environ["NOCOLOR"]
# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
# need to be inherited by ebuild subprocesses.
if 'PORTAGE_USERNAME' in os.environ:
make_conf['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
if 'PORTAGE_GRPNAME' in os.environ:
make_conf['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
make_conf_lines = []
for k_v in make_conf.items():
make_conf_lines.append('%s="%s"' % k_v)
if "make.conf" in user_config:
make_conf_lines.extend(user_config["make.conf"])
if not portage.process.sandbox_capable or \
os.environ.get("SANDBOX_ON") == "1":
# avoid problems from nested sandbox instances
make_conf_lines.append(
'FEATURES="${FEATURES} -sandbox -usersandbox"')
configs = user_config.copy()
configs["make.conf"] = make_conf_lines
for config_file, lines in configs.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(user_config_dir, config_file)
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
#Create /usr/share/portage/config/make.globals
make_globals_path = os.path.join(self.eroot,
GLOBAL_CONFIG_PATH.lstrip(os.sep),
"make.globals")
ensure_dirs(os.path.dirname(make_globals_path))
os.symlink(os.path.join(cnf_path, "make.globals"), make_globals_path)
#Create /usr/share/portage/config/sets/portage.conf
default_sets_conf_dir = os.path.join(self.eroot,
"usr/share/portage/config/sets")
try:
os.makedirs(default_sets_conf_dir)
except os.error:
pass
provided_sets_portage_conf = (os.path.join(cnf_path, "sets",
"portage.conf"))
os.symlink(provided_sets_portage_conf,
os.path.join(default_sets_conf_dir, "portage.conf"))
set_config_dir = os.path.join(user_config_dir, "sets")
try:
os.makedirs(set_config_dir)
except os.error:
pass
for sets_file, lines in sets.items():
file_name = os.path.join(set_config_dir, sets_file)
with open(file_name, "w") as f:
for line in lines:
f.write("%s\n" % line)
def digestgen(myarchives=None,
mysettings=None,
overwrite=None,
manifestonly=None,
myportdb=None):
"""
Generates a digest file if missing. Fetches files if necessary.
NOTE: myarchives and mysettings used to be positional arguments,
so their order must be preserved for backward compatibility.
@param mysettings: the ebuild config (mysettings["O"] must correspond
to the ebuild's parent directory)
@type mysettings: config
@param myportdb: a portdbapi instance
@type myportdb: portdbapi
@rtype: int
@returns: 1 on success and 0 on failure
"""
if mysettings is None:
raise TypeError("portage.digestgen(): missing" + \
" required 'mysettings' parameter")
if myportdb is None:
warnings.warn(
"portage.digestgen() called without 'myportdb' parameter",
DeprecationWarning,
stacklevel=2)
myportdb = portage.portdb
if overwrite is not None:
warnings.warn("portage.digestgen() called with " + \
"deprecated 'overwrite' parameter",
DeprecationWarning, stacklevel=2)
if manifestonly is not None:
warnings.warn("portage.digestgen() called with " + \
"deprecated 'manifestonly' parameter",
DeprecationWarning, stacklevel=2)
try:
portage._doebuild_manifest_exempt_depend += 1
distfiles_map = {}
fetchlist_dict = FetchlistDict(mysettings["O"], mysettings, myportdb)
for cpv in fetchlist_dict:
try:
for myfile in fetchlist_dict[cpv]:
distfiles_map.setdefault(myfile, []).append(cpv)
except InvalidDependString as e:
writemsg("!!! %s\n" % str(e), noiselevel=-1)
del e
return 0
mytree = os.path.dirname(os.path.dirname(mysettings["O"]))
manifest1_compat = False
mf = Manifest(mysettings["O"],
mysettings["DISTDIR"],
fetchlist_dict=fetchlist_dict,
manifest1_compat=manifest1_compat)
# Don't require all hashes since that can trigger excessive
# fetches when sufficient digests already exist. To ease transition
# while Manifest 1 is being removed, only require hashes that will
# exist before and after the transition.
required_hash_types = set()
required_hash_types.add("size")
required_hash_types.add(MANIFEST2_REQUIRED_HASH)
dist_hashes = mf.fhashdict.get("DIST", {})
# To avoid accidental regeneration of digests with the incorrect
# files (such as partially downloaded files), trigger the fetch
# code if the file exists and it's size doesn't match the current
# manifest entry. If there really is a legitimate reason for the
# digest to change, `ebuild --force digest` can be used to avoid
# triggering this code (or else the old digests can be manually
# removed from the Manifest).
missing_files = []
for myfile in distfiles_map:
myhashes = dist_hashes.get(myfile)
if not myhashes:
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError:
st = None
if st is None or st.st_size == 0:
missing_files.append(myfile)
continue
size = myhashes.get("size")
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError as e:
if e.errno != errno.ENOENT:
raise
del e
if size == 0:
missing_files.append(myfile)
continue
if required_hash_types.difference(myhashes):
missing_files.append(myfile)
continue
else:
if st.st_size == 0 or size is not None and size != st.st_size:
missing_files.append(myfile)
continue
if missing_files:
mytree = os.path.realpath(
os.path.dirname(os.path.dirname(mysettings["O"])))
fetch_settings = config(clone=mysettings)
debug = mysettings.get("PORTAGE_DEBUG") == "1"
for myfile in missing_files:
uris = set()
for cpv in distfiles_map[myfile]:
myebuild = os.path.join(mysettings["O"],
catsplit(cpv)[1] + ".ebuild")
# for RESTRICT=fetch, mirror, etc...
doebuild_environment(myebuild, "fetch", mysettings["ROOT"],
fetch_settings, debug, 1, myportdb)
uris.update(
myportdb.getFetchMap(cpv, mytree=mytree)[myfile])
fetch_settings["A"] = myfile # for use by pkg_nofetch()
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError:
st = None
if not fetch({myfile: uris}, fetch_settings):
writemsg(_("!!! Fetch failed for %s, can't update "
"Manifest\n") % myfile,
noiselevel=-1)
if myfile in dist_hashes and \
st is not None and st.st_size > 0:
# stat result is obtained before calling fetch(),
# since fetch may rename the existing file if the
# digest does not match.
writemsg(_("!!! If you would like to "
"forcefully replace the existing "
"Manifest entry\n!!! for %s, use "
"the following command:\n") % myfile + \
"!!! " + colorize("INFORM",
"ebuild --force %s manifest" % \
os.path.basename(myebuild)) + "\n",
noiselevel=-1)
return 0
writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"])
try:
mf.create(assumeDistHashesSometimes=True,
assumeDistHashesAlways=("assume-digests"
in mysettings.features))
except FileNotFound as e:
writemsg(_("!!! File %s doesn't exist, can't update "
"Manifest\n") % e,
noiselevel=-1)
return 0
except PortagePackageException as e:
writemsg(("!!! %s\n") % (e, ), noiselevel=-1)
return 0
try:
mf.write(sign=False)
except PermissionDenied as e:
writemsg(_("!!! Permission Denied: %s\n") % (e, ), noiselevel=-1)
return 0
if "assume-digests" not in mysettings.features:
distlist = list(mf.fhashdict.get("DIST", {}))
distlist.sort()
auto_assumed = []
for filename in distlist:
if not os.path.exists(
os.path.join(mysettings["DISTDIR"], filename)):
auto_assumed.append(filename)
if auto_assumed:
mytree = os.path.realpath(
os.path.dirname(os.path.dirname(mysettings["O"])))
cp = os.path.sep.join(mysettings["O"].split(os.path.sep)[-2:])
pkgs = myportdb.cp_list(cp, mytree=mytree)
pkgs.sort()
writemsg_stdout(" digest.assumed" +
colorize("WARN",
str(len(auto_assumed)).rjust(18)) +
"\n")
for pkg_key in pkgs:
fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree)
pv = pkg_key.split("/")[1]
for filename in auto_assumed:
if filename in fetchlist:
writemsg_stdout(" %s::%s\n" % (pv, filename))
return 1
finally:
portage._doebuild_manifest_exempt_depend -= 1
def _create_profile(self, ebuilds, installed, profile, user_config, sets):
#Create $PORTDIR/profiles/categories
categories = set()
for cpv in chain(ebuilds.keys(), installed.keys()):
categories.add(catsplit(cpv)[0])
profile_dir = os.path.join(self.portdir, "profiles")
try:
os.makedirs(profile_dir)
except os.error:
pass
categories_file = os.path.join(profile_dir, "categories")
f = open(categories_file, "w")
for cat in categories:
f.write(cat + "\n")
f.close()
#Create $REPO/profiles/license_groups
license_file = os.path.join(profile_dir, "license_groups")
f = open(license_file, "w")
f.write("EULA TEST\n")
f.close()
#Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
os.makedirs(os.path.join(self.portdir, "eclass"))
sub_profile_dir = os.path.join(profile_dir, "default", "linux", "x86", "test_profile")
os.makedirs(sub_profile_dir)
eapi_file = os.path.join(sub_profile_dir, "eapi")
f = open(eapi_file, "w")
f.write("0\n")
f.close()
make_defaults_file = os.path.join(sub_profile_dir, "make.defaults")
f = open(make_defaults_file, "w")
f.write("ARCH=\"x86\"\n")
f.write("ACCEPT_KEYWORDS=\"x86\"\n")
f.close()
use_force_file = os.path.join(sub_profile_dir, "use.force")
f = open(use_force_file, "w")
f.write("x86\n")
f.close()
if profile:
#This is meant to allow the consumer to set up his own profile,
#with package.mask and what not.
raise NotImplementedError()
#Create profile symlink
os.makedirs(os.path.join(self.eroot, "etc"))
os.symlink(sub_profile_dir, os.path.join(self.eroot, "etc", "make.profile"))
user_config_dir = os.path.join(self.eroot, "etc", "portage")
try:
os.makedirs(user_config_dir)
except os.error:
pass
for config_file, lines in user_config.items():
if config_file not in self.config_files:
raise ValueError("Unknown config file: '%s'" % config_file)
file_name = os.path.join(user_config_dir, config_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
#Create /usr/share/portage/config/sets/portage.conf
default_sets_conf_dir = os.path.join(self.eroot, "usr/share/portage/config/sets")
try:
os.makedirs(default_sets_conf_dir)
except os.error:
pass
provided_sets_portage_conf = \
os.path.join(PORTAGE_BASE_PATH, "cnf/sets/portage.conf")
os.symlink(provided_sets_portage_conf, os.path.join(default_sets_conf_dir, "portage.conf"))
set_config_dir = os.path.join(user_config_dir, "sets")
try:
os.makedirs(set_config_dir)
except os.error:
pass
for sets_file, lines in sets.items():
file_name = os.path.join(set_config_dir, sets_file)
f = open(file_name, "w")
for line in lines:
f.write("%s\n" % line)
f.close()
