def revisionMatch(revisionAtom, dbapi, match_type="default"): """ handler for the special >~, >=~, <=~ and <~ atoms that are supposed to behave as > and < except that they are limited to the same version, the range only applies to the revision part. @type revisionAtom: string @param revisionAtom: a <~ or >~ atom that contains the atom to match against @type dbapi: portage.dbapi.dbapi @param dbapi: one of the portage databases to use as information source @type match_type: string @param match_type: if != "default" passed as first argument to portdb.xmatch to apply the wanted visibility filters @rtype: list of strings @return: a list with the matching versions """ if match_type == "default" or not hasattr(dbapi, "xmatch"): if ":" in revisionAtom: mylist = dbapi.match(re.sub(r'-r[0-9]+(:[^ ]+)?$', r'\1', revisionAtom[2:])) else: mylist = dbapi.match(re.sub("-r[0-9]+$", "", revisionAtom[2:])) else: if ":" in revisionAtom: mylist = dbapi.xmatch(match_type, re.sub(r'-r[0-9]+(:[^ ]+)?$', r'\1', revisionAtom[2:])) else: mylist = dbapi.xmatch(match_type, re.sub("-r[0-9]+$", "", revisionAtom[2:])) rValue = [] for v in mylist: r1 = pkgsplit(v)[-1][1:] r2 = pkgsplit(revisionAtom[3:])[-1][1:] if eval(r1+" "+revisionAtom[0:2]+" "+r2): rValue.append(v) return rValue
def postbuild_success(self, build_info): """ Analyze compilation logs """ statusdict = dict(NO_WALL=list(), NO_WEXTRA=list()) target = os.path.join( '/usr/targets', os.getenv('CURRENT_TARGET', os.path.basename(self.cfg['build']['workdir'])), 'root' ) myconfig = config(target_root=target, config_root=target) for item in myconfig.packages: if item.startswith('*'): continue cp, v, r = pkgsplit(item[1:]) noflags = self.validateLogfile(cp, myconfig, os.path.basename(self.cfg['build']['workdir'])) if noflags: s = '%s-%s%s' % (cp, v, '-%s' % r if r != 'r0' else '') for k in noflags: statusdict[k].append(s) report = ['#. Log file analysis report\n\n'] if statusdict['NO_WALL']: report.append( "Packages which do not have '-Wall' compiler flag enabled:\n\n" + ''.join('\t* %s\n' % s for s in statusdict['NO_WALL']) ) if statusdict['NO_WEXTRA']: report.append( "Packages which do not have '-Wextra' compiler flag enabled:\n\n" + ''.join('\t* %s\n' % s for s in statusdict['NO_WEXTRA']) ) if len(report) == 1: report.append('Congratulations: everything is perfect!') build_info['analyzer'] = ''.join(report)
def check_locuse(portdir, pkg, invalid): locuse = [] ppkg = pkgsplit(strip_atoms(pkg)) if ppkg: ppkg = ppkg[0] else: ppkg = strip_atoms(pkg) metadata = join(portdir, ppkg, "metadata.xml") tree = etree.parse(metadata) root = tree.getroot() for elem in root: if elem.tag == "use": for use in elem: locuse.append(use.get("name")) # create a _NEW_ list oldinvalid = [foo for foo in invalid] for iuse in oldinvalid: if iuse in locuse: invalid.remove(iuse) return invalid
def gentoo_version(self): if not self.error: portage = portagetree() versions = portage.dep_match(self.atom) if len(versions) == 0: raise Exception('package not found %r' % self.atom) last_atom = versions[-1] cp, pv, rev = pkgsplit(last_atom) return pv
def __lt__(self, other): """ Compare different file names, first by file type and then for ebuilds by version and lexicographically for others. EBUILD < MISC < AUX < DIST < None """ if self.__class__ != other.__class__: raise NotImplementedError # Sort by file type as defined by _file_type_lt(). if self._file_type_lt(self, other): return True elif self._file_type_lt(other, self): return False # Files have the same type. if self.file_type == "EBUILD": # Sort by version. Lowest first. ver = "-".join(pkgsplit(self.file_name[:-7])[1:3]) other_ver = "-".join(pkgsplit(other.file_name[:-7])[1:3]) return vercmp(ver, other_ver) < 0 else: # Sort lexicographically. return self.file_name < other.file_name
def __lt__(self, other): """ Compare different file names, first by file type and then for ebuilds by version and lexicographically for others. EBUILD < MISC < AUX < DIST < None """ if self.__class__ != other.__class__: raise NotImplementedError # Sort by file type as defined by _file_type_lt(). if self._file_type_lt(self, other): return True if self._file_type_lt(other, self): return False # Files have the same type. if self.file_type == "EBUILD": # Sort by version. Lowest first. ver = "-".join(pkgsplit(self.file_name[:-7])[1:3]) other_ver = "-".join(pkgsplit(other.file_name[:-7])[1:3]) return vercmp(ver, other_ver) < 0 # Sort lexicographically. return self.file_name < other.file_name
def __init__(self, vdbdir, cat=None, name=None): self.cat = cat self.name = name self.vdbdir = vdbdir self.pkgfiles = None self.uses = None self.licenses = None self.license_choosen = None self.public = False self.scm = None self.res = 0 pkg_split = pkgsplit(name) self.short_name = pkg_split[0] if len(pkg_split) >= 3 and pkg_split[2] == 'r0': self.version = pkg_split[1] else: self.version = '-'.join(pkg_split[1:]) self.load_pkg(vdbdir)
def check_pkg(portdir, line): # print "PKGM1:", line pkgm = line.split(" ")[0] # print "PKGM2:", pkgm # print "DBG:", line.split(" ") if pkgm.startswith("-"): pkgm = pkgm[1:] if pkgm.startswith(OPERATORS): pkg = [] # print "DBG1: %s" % pkgm plain_pkg = strip_atoms(pkgm) # print "DBG2: %s" % plain_pkg pkg = pkgsplit(plain_pkg) if not pkg: print >> stderr, "Error encountered during pkgsplit(), please contact [email protected] including the whole output!" print >> stderr, "1: %s; 2: %s" % (pkgm, plain_pkg) return 0 plain_pkg = strip_atoms(pkg[0]) if not isdir(join(portdir, plain_pkg)): return 0 if not pkgcmp_atom(join(portdir, plain_pkg), pkgm): return 0 return 1 else: if pkgm.find(":") != -1: pkgm = strip_atoms(pkgm) if isdir(join(portdir, pkgm)): return 1 else: return 0 return 0
if len(dirs) + len(files) == 0: print root, "seems to be empty" # find obsolete package.mask entries PACKAGE_MASK = join(os.getcwd(), "profiles/package.mask") TOCHECK = [] if not isfile(PACKAGE_MASK): print >> stderr, "can't find package.mask, you must be in the overlay root!" exit(1) PMASK = portage.env.config.PackageMaskFile(PACKAGE_MASK) PMASK.load() for key in PMASK.keys(): # remove =, >=, ~ etc. while not key[0].isalpha(): key = key[1:] # remove trailing * or . like in: ...-1.4.* or so while key[-1] == "*" or key[-1] == ".": key = key[:-1] if pkgsplit(key): TOCHECK.append(pkgsplit(key)[0]) else: TOCHECK.append(key) for item in TOCHECK: if not isdir(item): print "package.mask entry for '%s' is obsolete!" % str(item)
def available_portage_update(self, detected=False, init=False): """ Check if an update to portage is available. """ # TODO: be more verbose for debug ! name = 'available_portage_update' logger = logging.getLogger(f'{self.__logger_name}{name}::') logger.debug(f"Running with detected={detected}, init={init}") self.available = False self.latest = False self.current = False # First, any way get installed and latest current = vardbapi().match('portage')[0] latest = portdbapi().xmatch('bestmatch-visible', 'portage') # Then just compare # From site-packages/portage/versions.py # @param mypkg: either a pv or cpv # @return: # 1. None if input is invalid. # 2. (pn, ver, rev) if input is pv # 3. (cp, ver, rev) if input is a cpv result = pkgcmp(pkgsplit(latest), pkgsplit(current)) # From site-packages/portage/versions.py # Parameters: # pkg1 (list (example: ['test', '1.0', 'r1'])) - # package to compare with # pkg2 (list (example: ['test', '1.0', 'r1'])) - # package to compare againts # Returns: None or integer # None if package names are not the same # 1 if pkg1 is greater than pkg2 # -1 if pkg1 is less than pkg2 # 0 if pkg1 equals pkg2 if result == None or result == -1: msg = 'no result (package names are not the same ?!)' if result == -1: msg = ('the latest version available is lower than the' ' one installed...') logger.error("FAILED to compare versions when obtaining update " "informations for the portage package.") logger.error(f"Result is: {msg}") logger.error(f"Current portage version: {current}, latest:" f" {latest}.") # Return is ignored for the moment... # TODO ?? return # Split current version # as we don't know yet if latest > current split = pkgsplit(current) self.current = split[1] if not split[2] == 'r0': self.current = '-'.join(split[-2:]) # Check if an update to portage is available if result: # Now, split latest because > current split = pkgsplit(latest) self.latest = split[1] if not split[2] == 'r0': self.latest = '-'.join(split[-2:]) logger.debug(f"Found an update to portage (from {self.current}" f" to {self.latest}).") # Print only one time when program start if init: logger.info("Found an update to portage (from " f"{self.current} to {self.latest}).") self.available = True else: logger.debug("No update to portage package is available" f" (current version: {self.current})") self.available = False # For detected we have to compare current extracted # version and last current know version (so from # self.portage['current']. Both versions are already # split if detected: # From site-packages/portage/versions.py # Compare two versions # Example usage: # >>> from portage.versions import vercmp # >>> vercmp('1.0-r1','1.2-r3') # negative number # >>> vercmp('1.3','1.2-r3') # positive number # >>> vercmp('1.0_p3','1.0_p3') # 0 # @param pkg1: version to compare with # (see ver_regexp in portage.versions.py) # @type pkg1: string (example: "2.1.2-r3") # @param pkg2: version to compare againts # (see ver_regexp in portage.versions.py) # @type pkg2: string (example: "2.1.2_rc5") # @rtype: None or float # @return: # 1. positive if ver1 is greater than ver2 # 2. negative if ver1 is less than ver2 # 3. 0 if ver1 equals ver2 # 4. None if ver1 or ver2 are invalid # (see ver_regexp in portage.versions.py) compare = vercmp(self.portage['current'], self.current) msg = False add_msg = '' if compare < 0: # If not available and it have been updated # than it have been updated to latest one if not self.available: add_msg = 'latest ' msg = (f"The portage package has been updated (from " f"{self.portage['current']} to " f"{add_msg}{self.current}).") elif compare > 0: # Same here but reversed: if it was not # available (self.portage['available']) # and now it is (self.available) than # it have been downgraded from latest. if not self.portage['available'] and self.available: add_msg = 'latest ' msg = (f"The portage package has been downgraded (from " f"{add_msg}{self.portage['current']} to " f"{self.current}).") elif compare == 0: # This have been aborted msg = ("The portage package process has been aborted.") # Just skipp if msg = False # so that mean compare == None if msg: logger.info(msg) tosave = [] # Update only if change for key in 'current', 'latest', 'available': if not self.portage[key] == getattr(self, key): # This print if there a new version of portage available # even if there is already an older version available # TEST: if checking only for key latest than it could # be == to current so check also result. if key == 'latest' and result: logger.info("Found an update to portage (from " f"{self.current} to {self.latest}).") self.portage[key] = getattr(self, key) tosave.append([f'portage {key}', self.portage[key]]) if tosave: self.stateinfo.save(*tosave)
def pkgcmp_atom(pkgdir, pkg): ebuilds = [] for ent in listdir(pkgdir): if ent.endswith(".ebuild"): ebuilds.append(ent) ppkg = basename(strip_atoms(pkg)) revre = re.compile( ("^" + re.escape(ppkg) + "(-r\d+)?.ebuild$") ) # print "DBG: checking for %s" % pkg # print "DBG: Got %i ebuilds:" % len(ebuilds) # print ebuilds for ebuild in ebuilds: # workaround? for - prefix if pkg.startswith( "-" ): pkg = pkg[1:] if pkg.startswith( ("=", "~") ): if pkg.startswith("~"): if revre.match(ebuild): # print "DBG: revmatch '%s' '%s'" % (pkg, ebuild) return 1 else: # print "DBG: revmatch continue" continue if pkg.endswith("*"): if ebuild.startswith(ppkg): # print "DBG: startswith '%s' '%s'" % (pkg, ebuild) return 1 else: # print "DBG: startswith continue" continue else: if ebuild == (ppkg + ".ebuild"): # print "DBG: '%s' == '%s'" % (ppkg, ppkg) return 1 else: # print "DBG: == continue" continue if pkg.startswith( (">=", ">", "<=", "<") ): plain = strip_atoms(pkg) mypkg = pkgsplit(plain) ourpkg = pkgsplit(ebuild.rstrip(".ebuild")) mypkgv = mypkg[1] if mypkg[2] != "r0": mypkgv = mypkgv + "-" + mypkg[2] ourpkgv = ourpkg[1] if ourpkg[2] != "r0": ourpkgv = ourpkgv + "-" + ourpkg[2] # print "MYPKGV:", mypkgv, "OURPKGV:", ourpkgv, "RESULT 'vercmp('%s', '%s'): %i" % (mypkgv, ourpkgv, vercmp(mypkgv, ourpkgv)) if pkg.startswith(">="): if vercmp(mypkgv, ourpkgv) <= 0: # print "HIT: '%s' >= '%s'" % (ourpkg, mypkg) return 1 else: # print ">= continue" continue if pkg.startswith(">") and not pkg.startswith(">="): if vercmp(mypkgv, ourpkgv) < 0: # print "HIT: '%s' > '%s'" % (ourpkg, mypkg) return 1 else: # print "> continue" continue if pkg.startswith("<="): if vercmp(mypkgv, ourpkgv) >= 0: # print "HIT: '%s' <= '%s'" % (ourpkg, mypkg) return 1 else: # print "<= continue" continue if pkg.startswith("<") and not pkg.startswith("<="): if vercmp(mypkgv, ourpkgv) > 0: # print "HIT: '%s' < '%s'" % (ourpkg, mypkg) return 1 else: # print "< continue" continue # print "Nothing found... '%s' is invalid" % pkg return 0
def revision(self): rs = pkgsplit(self._cpv)[2] assert (rs.startswith('r')) return int(rs[1:])
def without_revision(self): return pkgsplit(self._cpv)[1]
print root, "seems to be empty" # find obsolete package.mask entries PACKAGE_MASK = join(os.getcwd(), "profiles/package.mask") TOCHECK = [] if not isfile(PACKAGE_MASK): print >> stderr, "can't find package.mask, you must be in the overlay root!" exit(1) PMASK = portage.env.config.PackageMaskFile(PACKAGE_MASK) PMASK.load() for key in PMASK.keys(): # remove =, >=, ~ etc. while not key[0].isalpha(): key = key[1:] # remove trailing * or . like in: ...-1.4.* or so while key[-1] == "*" or key[-1] == ".": key = key[:-1] if pkgsplit(key): TOCHECK.append(pkgsplit(key)[0]) else: TOCHECK.append(key) for item in TOCHECK: if not isdir(item): print "package.mask entry for '%s' is obsolete!" % str(item)
def main(): argp = argparse.ArgumentParser() argp.add_argument("--targets", type=str, action="append", default=[ "//tensorflow/tools/pip_package:build_pip_package", "//tensorflow:libtensorflow.so", "@io_bazel_rules_closure//closure:defs.bzl" ], help="Targets to extract dependencies from") argp.add_argument("--repository_rules", type=str, action="append", default=[ "filegroup_external", "http_archive", "http_file", "new_http_archive", "patched_http_archive", "temp_workaround_http_archive" ], help="Repository rule types to include") argp.add_argument("ebuild", type=str, help="Ebuild file to update") argp.add_argument("workspace", type=str, help="Bazel workspace directory") args = argp.parse_args() if not os.path.exists(args.ebuild): raise ValueError("ebuild file does not exist") pn, pv, r = versions.pkgsplit(os.path.basename(args.ebuild)[:-7]) os.environ.update( dict(CC_OPT_FLAGS="-mnative -msse -msse2 -msse3", TF_NEED_JEMALLOC="1", TF_NEED_GCP="1", TF_NEED_HDFS="1", TF_NEED_MKL="0", TF_NEED_MPI="0", TF_NEED_OPENCL="0", TF_NEED_CUDA="1", TF_NEED_VERBS="0", GCC_HOST_COMPILER_PATH="/usr/bin/gcc", TF_CUDA_VERSION="8.0", TF_CUDA_CLANG="0", CUDA_TOOLKIT_PATH="/opt/cuda", TF_CUDNN_VERSION="6", CUDNN_INSTALL_PATH="/usr", TF_CUDA_COMPUTE_CAPABILITIES="3.5,5.2,6.1", TF_ENABLE_XLA="0", PYTHON_BIN_PATH="/usr/bin/python3.4", PYTHON_LIB_PATH="/usr/lib64/python3.4/site-packages")) subprocess.check_call(["./configure"], cwd=args.workspace) # Get external repositories. subprocess.check_call(["bazel", "fetch", "--config=opt"] + args.targets, cwd=args.workspace) deps = get_externals(args.workspace, args.targets, args.repository_rules) # Update the ebuild source URIs. update_ebuild_file(deps, args.ebuild) # Create a patch file with markers. filesdir = os.path.join(os.path.dirname(args.ebuild), "files") if not os.path.exists(filesdir): os.makedirs(filesdir) write_markers_file( deps, os.path.join(filesdir, "bazel-markers-{}.patch".format(pv)))