def revisionMatch(revisionAtom, dbapi, match_type="default"):
"""
handler for the special >~, >=~, <=~ and <~ atoms that are supposed to behave
as > and < except that they are limited to the same version, the range only
applies to the revision part.
@type revisionAtom: string
@param revisionAtom: a <~ or >~ atom that contains the atom to match against
@type dbapi: portage.dbapi.dbapi
@param dbapi: one of the portage databases to use as information source
@type match_type: string
@param match_type: if != "default" passed as first argument to portdb.xmatch
to apply the wanted visibility filters
@rtype: list of strings
@return: a list with the matching versions
"""
if match_type == "default" or not hasattr(dbapi, "xmatch"):
if ":" in revisionAtom:
mylist = dbapi.match(re.sub(r'-r[0-9]+(:[^ ]+)?$', r'\1', revisionAtom[2:]))
else:
mylist = dbapi.match(re.sub("-r[0-9]+$", "", revisionAtom[2:]))
else:
if ":" in revisionAtom:
mylist = dbapi.xmatch(match_type, re.sub(r'-r[0-9]+(:[^ ]+)?$', r'\1', revisionAtom[2:]))
else:
mylist = dbapi.xmatch(match_type, re.sub("-r[0-9]+$", "", revisionAtom[2:]))
rValue = []
for v in mylist:
r1 = pkgsplit(v)[-1][1:]
r2 = pkgsplit(revisionAtom[3:])[-1][1:]
if eval(r1+" "+revisionAtom[0:2]+" "+r2):
rValue.append(v)
return rValue
def revisionMatch(revisionAtom, dbapi, match_type="default"):
"""
handler for the special >~, >=~, <=~ and <~ atoms that are supposed to behave
as > and < except that they are limited to the same version, the range only
applies to the revision part.
@type revisionAtom: string
@param revisionAtom: a <~ or >~ atom that contains the atom to match against
@type dbapi: portage.dbapi.dbapi
@param dbapi: one of the portage databases to use as information source
@type match_type: string
@param match_type: if != "default" passed as first argument to portdb.xmatch
to apply the wanted visibility filters
@rtype: list of strings
@return: a list with the matching versions
"""
if match_type == "default" or not hasattr(dbapi, "xmatch"):
if ":" in revisionAtom:
mylist = dbapi.match(re.sub(r'-r[0-9]+(:[^ ]+)?$', r'\1', revisionAtom[2:]))
else:
mylist = dbapi.match(re.sub("-r[0-9]+$", "", revisionAtom[2:]))
else:
if ":" in revisionAtom:
mylist = dbapi.xmatch(match_type, re.sub(r'-r[0-9]+(:[^ ]+)?$', r'\1', revisionAtom[2:]))
else:
mylist = dbapi.xmatch(match_type, re.sub("-r[0-9]+$", "", revisionAtom[2:]))
rValue = []
for v in mylist:
r1 = pkgsplit(v)[-1][1:]
r2 = pkgsplit(revisionAtom[3:])[-1][1:]
if eval(r1+" "+revisionAtom[0:2]+" "+r2):
rValue.append(v)
return rValue
def postbuild_success(self, build_info):
""" Analyze compilation logs
"""
statusdict = dict(NO_WALL=list(), NO_WEXTRA=list())
target = os.path.join(
'/usr/targets', os.getenv('CURRENT_TARGET', os.path.basename(self.cfg['build']['workdir'])), 'root'
)
myconfig = config(target_root=target, config_root=target)
for item in myconfig.packages:
if item.startswith('*'):
continue
cp, v, r = pkgsplit(item[1:])
noflags = self.validateLogfile(cp, myconfig, os.path.basename(self.cfg['build']['workdir']))
if noflags:
s = '%s-%s%s' % (cp, v, '-%s' % r if r != 'r0' else '')
for k in noflags:
statusdict[k].append(s)
report = ['#. Log file analysis report\n\n']
if statusdict['NO_WALL']:
report.append(
"Packages which do not have '-Wall' compiler flag enabled:\n\n" +
''.join('\t* %s\n' % s for s in statusdict['NO_WALL'])
)
if statusdict['NO_WEXTRA']:
report.append(
"Packages which do not have '-Wextra' compiler flag enabled:\n\n" +
''.join('\t* %s\n' % s for s in statusdict['NO_WEXTRA'])
)
if len(report) == 1:
report.append('Congratulations: everything is perfect!')
build_info['analyzer'] = ''.join(report)
def check_locuse(portdir, pkg, invalid):
locuse = []
ppkg = pkgsplit(strip_atoms(pkg))
if ppkg:
ppkg = ppkg[0]
else:
ppkg = strip_atoms(pkg)
metadata = join(portdir, ppkg, "metadata.xml")
tree = etree.parse(metadata)
root = tree.getroot()
for elem in root:
if elem.tag == "use":
for use in elem:
locuse.append(use.get("name"))
# create a _NEW_ list
oldinvalid = [foo for foo in invalid]
for iuse in oldinvalid:
if iuse in locuse:
invalid.remove(iuse)
return invalid
def gentoo_version(self):
if not self.error:
portage = portagetree()
versions = portage.dep_match(self.atom)
if len(versions) == 0:
raise Exception('package not found %r' % self.atom)
last_atom = versions[-1]
cp, pv, rev = pkgsplit(last_atom)
return pv
def __lt__(self, other): """ Compare different file names, first by file type and then for ebuilds by version and lexicographically for others. EBUILD < MISC < AUX < DIST < None """ if self.__class__ != other.__class__: raise NotImplementedError # Sort by file type as defined by _file_type_lt(). if self._file_type_lt(self, other): return True elif self._file_type_lt(other, self): return False # Files have the same type. if self.file_type == "EBUILD": # Sort by version. Lowest first. ver = "-".join(pkgsplit(self.file_name[:-7])[1:3]) other_ver = "-".join(pkgsplit(other.file_name[:-7])[1:3]) return vercmp(ver, other_ver) < 0 else: # Sort lexicographically. return self.file_name < other.file_name
def __lt__(self, other):
"""
Compare different file names, first by file type and then
for ebuilds by version and lexicographically for others.
EBUILD < MISC < AUX < DIST < None
"""
if self.__class__ != other.__class__:
raise NotImplementedError
# Sort by file type as defined by _file_type_lt().
if self._file_type_lt(self, other):
return True
if self._file_type_lt(other, self):
return False
# Files have the same type.
if self.file_type == "EBUILD":
# Sort by version. Lowest first.
ver = "-".join(pkgsplit(self.file_name[:-7])[1:3])
other_ver = "-".join(pkgsplit(other.file_name[:-7])[1:3])
return vercmp(ver, other_ver) < 0
# Sort lexicographically.
return self.file_name < other.file_name
def __init__(self, vdbdir, cat=None, name=None):
self.cat = cat
self.name = name
self.vdbdir = vdbdir
self.pkgfiles = None
self.uses = None
self.licenses = None
self.license_choosen = None
self.public = False
self.scm = None
self.res = 0
pkg_split = pkgsplit(name)
self.short_name = pkg_split[0]
if len(pkg_split) >= 3 and pkg_split[2] == 'r0':
self.version = pkg_split[1]
else:
self.version = '-'.join(pkg_split[1:])
self.load_pkg(vdbdir)
def check_pkg(portdir, line):
# print "PKGM1:", line
pkgm = line.split(" ")[0]
# print "PKGM2:", pkgm
# print "DBG:", line.split(" ")
if pkgm.startswith("-"):
pkgm = pkgm[1:]
if pkgm.startswith(OPERATORS):
pkg = []
# print "DBG1: %s" % pkgm
plain_pkg = strip_atoms(pkgm)
# print "DBG2: %s" % plain_pkg
pkg = pkgsplit(plain_pkg)
if not pkg:
print >> stderr, "Error encountered during pkgsplit(), please contact [email protected] including the whole output!"
print >> stderr, "1: %s; 2: %s" % (pkgm, plain_pkg)
return 0
plain_pkg = strip_atoms(pkg[0])
if not isdir(join(portdir, plain_pkg)):
return 0
if not pkgcmp_atom(join(portdir, plain_pkg), pkgm):
return 0
return 1
else:
if pkgm.find(":") != -1:
pkgm = strip_atoms(pkgm)
if isdir(join(portdir, pkgm)):
return 1
else:
return 0
return 0
if len(dirs) + len(files) == 0:
print root, "seems to be empty"
# find obsolete package.mask entries
PACKAGE_MASK = join(os.getcwd(), "profiles/package.mask")
TOCHECK = []
if not isfile(PACKAGE_MASK):
print >> stderr, "can't find package.mask, you must be in the overlay root!"
exit(1)
PMASK = portage.env.config.PackageMaskFile(PACKAGE_MASK)
PMASK.load()
for key in PMASK.keys():
# remove =, >=, ~ etc.
while not key[0].isalpha():
key = key[1:]
# remove trailing * or . like in: ...-1.4.* or so
while key[-1] == "*" or key[-1] == ".":
key = key[:-1]
if pkgsplit(key):
TOCHECK.append(pkgsplit(key)[0])
else:
TOCHECK.append(key)
for item in TOCHECK:
if not isdir(item):
print "package.mask entry for '%s' is obsolete!" % str(item)
def available_portage_update(self, detected=False, init=False):
"""
Check if an update to portage is available.
"""
# TODO: be more verbose for debug !
name = 'available_portage_update'
logger = logging.getLogger(f'{self.__logger_name}{name}::')
logger.debug(f"Running with detected={detected}, init={init}")
self.available = False
self.latest = False
self.current = False
# First, any way get installed and latest
current = vardbapi().match('portage')[0]
latest = portdbapi().xmatch('bestmatch-visible', 'portage')
# Then just compare
# From site-packages/portage/versions.py
# @param mypkg: either a pv or cpv
# @return:
# 1. None if input is invalid.
# 2. (pn, ver, rev) if input is pv
# 3. (cp, ver, rev) if input is a cpv
result = pkgcmp(pkgsplit(latest), pkgsplit(current))
# From site-packages/portage/versions.py
# Parameters:
# pkg1 (list (example: ['test', '1.0', 'r1'])) -
# package to compare with
# pkg2 (list (example: ['test', '1.0', 'r1'])) -
# package to compare againts
# Returns: None or integer
# None if package names are not the same
# 1 if pkg1 is greater than pkg2
# -1 if pkg1 is less than pkg2
# 0 if pkg1 equals pkg2
if result == None or result == -1:
msg = 'no result (package names are not the same ?!)'
if result == -1:
msg = ('the latest version available is lower than the'
' one installed...')
logger.error("FAILED to compare versions when obtaining update "
"informations for the portage package.")
logger.error(f"Result is: {msg}")
logger.error(f"Current portage version: {current}, latest:"
f" {latest}.")
# Return is ignored for the moment...
# TODO ??
return
# Split current version
# as we don't know yet if latest > current
split = pkgsplit(current)
self.current = split[1]
if not split[2] == 'r0':
self.current = '-'.join(split[-2:])
# Check if an update to portage is available
if result:
# Now, split latest because > current
split = pkgsplit(latest)
self.latest = split[1]
if not split[2] == 'r0':
self.latest = '-'.join(split[-2:])
logger.debug(f"Found an update to portage (from {self.current}"
f" to {self.latest}).")
# Print only one time when program start
if init:
logger.info("Found an update to portage (from "
f"{self.current} to {self.latest}).")
self.available = True
else:
logger.debug("No update to portage package is available"
f" (current version: {self.current})")
self.available = False
# For detected we have to compare current extracted
# version and last current know version (so from
# self.portage['current']. Both versions are already
# split
if detected:
# From site-packages/portage/versions.py
# Compare two versions
# Example usage:
# >>> from portage.versions import vercmp
# >>> vercmp('1.0-r1','1.2-r3')
# negative number
# >>> vercmp('1.3','1.2-r3')
# positive number
# >>> vercmp('1.0_p3','1.0_p3')
# 0
# @param pkg1: version to compare with
# (see ver_regexp in portage.versions.py)
# @type pkg1: string (example: "2.1.2-r3")
# @param pkg2: version to compare againts
# (see ver_regexp in portage.versions.py)
# @type pkg2: string (example: "2.1.2_rc5")
# @rtype: None or float
# @return:
# 1. positive if ver1 is greater than ver2
# 2. negative if ver1 is less than ver2
# 3. 0 if ver1 equals ver2
# 4. None if ver1 or ver2 are invalid
# (see ver_regexp in portage.versions.py)
compare = vercmp(self.portage['current'], self.current)
msg = False
add_msg = ''
if compare < 0:
# If not available and it have been updated
# than it have been updated to latest one
if not self.available:
add_msg = 'latest '
msg = (f"The portage package has been updated (from "
f"{self.portage['current']} to "
f"{add_msg}{self.current}).")
elif compare > 0:
# Same here but reversed: if it was not
# available (self.portage['available'])
# and now it is (self.available) than
# it have been downgraded from latest.
if not self.portage['available'] and self.available:
add_msg = 'latest '
msg = (f"The portage package has been downgraded (from "
f"{add_msg}{self.portage['current']} to "
f"{self.current}).")
elif compare == 0:
# This have been aborted
msg = ("The portage package process has been aborted.")
# Just skipp if msg = False
# so that mean compare == None
if msg:
logger.info(msg)
tosave = []
# Update only if change
for key in 'current', 'latest', 'available':
if not self.portage[key] == getattr(self, key):
# This print if there a new version of portage available
# even if there is already an older version available
# TEST: if checking only for key latest than it could
# be == to current so check also result.
if key == 'latest' and result:
logger.info("Found an update to portage (from "
f"{self.current} to {self.latest}).")
self.portage[key] = getattr(self, key)
tosave.append([f'portage {key}', self.portage[key]])
if tosave:
self.stateinfo.save(*tosave)
def pkgcmp_atom(pkgdir, pkg):
ebuilds = []
for ent in listdir(pkgdir):
if ent.endswith(".ebuild"):
ebuilds.append(ent)
ppkg = basename(strip_atoms(pkg))
revre = re.compile( ("^" + re.escape(ppkg) + "(-r\d+)?.ebuild$") )
# print "DBG: checking for %s" % pkg
# print "DBG: Got %i ebuilds:" % len(ebuilds)
# print ebuilds
for ebuild in ebuilds:
# workaround? for - prefix
if pkg.startswith( "-" ):
pkg = pkg[1:]
if pkg.startswith( ("=", "~") ):
if pkg.startswith("~"):
if revre.match(ebuild):
# print "DBG: revmatch '%s' '%s'" % (pkg, ebuild)
return 1
else:
# print "DBG: revmatch continue"
continue
if pkg.endswith("*"):
if ebuild.startswith(ppkg):
# print "DBG: startswith '%s' '%s'" % (pkg, ebuild)
return 1
else:
# print "DBG: startswith continue"
continue
else:
if ebuild == (ppkg + ".ebuild"):
# print "DBG: '%s' == '%s'" % (ppkg, ppkg)
return 1
else:
# print "DBG: == continue"
continue
if pkg.startswith( (">=", ">", "<=", "<") ):
plain = strip_atoms(pkg)
mypkg = pkgsplit(plain)
ourpkg = pkgsplit(ebuild.rstrip(".ebuild"))
mypkgv = mypkg[1]
if mypkg[2] != "r0":
mypkgv = mypkgv + "-" + mypkg[2]
ourpkgv = ourpkg[1]
if ourpkg[2] != "r0":
ourpkgv = ourpkgv + "-" + ourpkg[2]
# print "MYPKGV:", mypkgv, "OURPKGV:", ourpkgv, "RESULT 'vercmp('%s', '%s'): %i" % (mypkgv, ourpkgv, vercmp(mypkgv, ourpkgv))
if pkg.startswith(">="):
if vercmp(mypkgv, ourpkgv) <= 0:
# print "HIT: '%s' >= '%s'" % (ourpkg, mypkg)
return 1
else:
# print ">= continue"
continue
if pkg.startswith(">") and not pkg.startswith(">="):
if vercmp(mypkgv, ourpkgv) < 0:
# print "HIT: '%s' > '%s'" % (ourpkg, mypkg)
return 1
else:
# print "> continue"
continue
if pkg.startswith("<="):
if vercmp(mypkgv, ourpkgv) >= 0:
# print "HIT: '%s' <= '%s'" % (ourpkg, mypkg)
return 1
else:
# print "<= continue"
continue
if pkg.startswith("<") and not pkg.startswith("<="):
if vercmp(mypkgv, ourpkgv) > 0:
# print "HIT: '%s' < '%s'" % (ourpkg, mypkg)
return 1
else:
# print "< continue"
continue
# print "Nothing found... '%s' is invalid" % pkg
return 0
def revision(self):
rs = pkgsplit(self._cpv)[2]
assert (rs.startswith('r'))
return int(rs[1:])
def without_revision(self):
return pkgsplit(self._cpv)[1]
print root, "seems to be empty" # find obsolete package.mask entries PACKAGE_MASK = join(os.getcwd(), "profiles/package.mask") TOCHECK = [] if not isfile(PACKAGE_MASK): print >> stderr, "can't find package.mask, you must be in the overlay root!" exit(1) PMASK = portage.env.config.PackageMaskFile(PACKAGE_MASK) PMASK.load() for key in PMASK.keys(): # remove =, >=, ~ etc. while not key[0].isalpha(): key = key[1:] # remove trailing * or . like in: ...-1.4.* or so while key[-1] == "*" or key[-1] == ".": key = key[:-1] if pkgsplit(key): TOCHECK.append(pkgsplit(key)[0]) else: TOCHECK.append(key) for item in TOCHECK: if not isdir(item): print "package.mask entry for '%s' is obsolete!" % str(item)
def main():
argp = argparse.ArgumentParser()
argp.add_argument("--targets",
type=str,
action="append",
default=[
"//tensorflow/tools/pip_package:build_pip_package",
"//tensorflow:libtensorflow.so",
"@io_bazel_rules_closure//closure:defs.bzl"
],
help="Targets to extract dependencies from")
argp.add_argument("--repository_rules",
type=str,
action="append",
default=[
"filegroup_external", "http_archive", "http_file",
"new_http_archive", "patched_http_archive",
"temp_workaround_http_archive"
],
help="Repository rule types to include")
argp.add_argument("ebuild", type=str, help="Ebuild file to update")
argp.add_argument("workspace", type=str, help="Bazel workspace directory")
args = argp.parse_args()
if not os.path.exists(args.ebuild):
raise ValueError("ebuild file does not exist")
pn, pv, r = versions.pkgsplit(os.path.basename(args.ebuild)[:-7])
os.environ.update(
dict(CC_OPT_FLAGS="-mnative -msse -msse2 -msse3",
TF_NEED_JEMALLOC="1",
TF_NEED_GCP="1",
TF_NEED_HDFS="1",
TF_NEED_MKL="0",
TF_NEED_MPI="0",
TF_NEED_OPENCL="0",
TF_NEED_CUDA="1",
TF_NEED_VERBS="0",
GCC_HOST_COMPILER_PATH="/usr/bin/gcc",
TF_CUDA_VERSION="8.0",
TF_CUDA_CLANG="0",
CUDA_TOOLKIT_PATH="/opt/cuda",
TF_CUDNN_VERSION="6",
CUDNN_INSTALL_PATH="/usr",
TF_CUDA_COMPUTE_CAPABILITIES="3.5,5.2,6.1",
TF_ENABLE_XLA="0",
PYTHON_BIN_PATH="/usr/bin/python3.4",
PYTHON_LIB_PATH="/usr/lib64/python3.4/site-packages"))
subprocess.check_call(["./configure"], cwd=args.workspace)
# Get external repositories.
subprocess.check_call(["bazel", "fetch", "--config=opt"] + args.targets,
cwd=args.workspace)
deps = get_externals(args.workspace, args.targets, args.repository_rules)
# Update the ebuild source URIs.
update_ebuild_file(deps, args.ebuild)
# Create a patch file with markers.
filesdir = os.path.join(os.path.dirname(args.ebuild), "files")
if not os.path.exists(filesdir):
os.makedirs(filesdir)
write_markers_file(
deps, os.path.join(filesdir, "bazel-markers-{}.patch".format(pv)))