def handleCheck(self, startGroups, stopGroups, lines, fullpath): if stopGroups is None: # we lost sync, don't start guessing because we care about # the result of the check return sought = startGroups[0] success = self.parseSuccess(stopGroups[0]) includeDirs = [ '%(includedir)s/' %self.macros] root = self.recipe.cfg.root if success: if self.headerRe.match(sought): # look for header files for tokens in (x.split() for x in lines): for token in tokens: if token.startswith('-I/') and len(token) > 3: includeDirs.append(token[2:]) for dirName in includeDirs: seekPath = util.normpath('%s/%s' %(dirName, sought)) if util.exists('%s%s' %(root, seekPath)): self.foundPaths.add(seekPath) break libName = self.libRe.match(sought) if libName: libName = libName.group(0) # Take advantage of the fact that the actual test will # include running the compiler with the library in the # link line in such a way that the # EnforceStaticLibBuildRequirements policy knows how # to understand it. # EnforceStaticLibBuildRequirements does not handle the # leading "configure:01234: " portion of the output, # so give it every line that has further content and # let it find the lines that it cares about logLines = (x.split(': ', 1) for x in lines) logLines = (x[1] for x in logLines if len(x) > 1) self.recipe.EnforceStaticLibBuildRequirements(logLines=logLines) candidate = None if sought.startswith('/'): candidate = sought elif type(success) is str and success.startswith('/'): candidate = success if candidate: # configure:2345: checking for /bin/sought # configure:6543: result: yes # configure:4441: checking for egrep # configure:4519: result: /bin/grep -E # configure:4535: checking for ld used by gcc # configure:4602: result: /usr/bin/ld seekPath = candidate.split()[0] if util.exists(util.normpath('%s%s' %(root, seekPath))): self.foundPaths.update(set( self.greylistFilter(set((seekPath,)), fullpath)))
def do(self): if not self.buildTestSuite: self.recipe.TestSuiteFiles(build=False) return # expand macros in fileMap newFileMap = {} for (buildfile, destfile) in self.fileMap.iteritems(): newFileMap[util.normpath(buildfile % self.macros)] = destfile % self.macros self.fileMap = newFileMap self.builddirfiles = {} self.builddirlinks = {} builddir = self.macros.builddir builddirlen = len(builddir) for root, dirs, files in os.walk(builddir): for file in files: realDir = root[builddirlen:] realPath = os.path.join(realDir, file) if realPath in self.fileMap: continue fullpath = os.path.join(root, file) if os.path.islink(fullpath): # symlink handling: # change to absolute link and add to symlink list contents = os.readlink(fullpath) if contents[0] != '/': contents = util.normpath(os.path.join( root, contents))[builddirlen:] if contents not in self.builddirlinks: self.builddirlinks[contents] = [] self.builddirlinks[contents].append( os.path.join(realDir, file)) else: # add to regular file list if file not in self.builddirfiles: self.builddirfiles[file] = [] self.builddirfiles[file].append(realDir) if self.buildTestSuite: for (buildfile, destfile) in self.fileMap.iteritems(): target = destfile link = util.normpath('%(destdir)s%(thistestdir)s/' % self.macros + buildfile) util.mkdirChain(os.path.dirname(link)) os.symlink(target, link) self.recipe.TestSuiteFiles(build=True) self.recipe.TestSuiteFiles(builddirlinks=self.builddirlinks) policy.Policy.do(self) else: self.recipe.TestSuiteFiles(build=False) return
def doFile(self, path): if hasattr(self.recipe, '_getCapsulePathsForFile'): if self.recipe._getCapsulePathsForFile(path): return d = self.macros.destdir destlen = len(d) l = util.joinPaths(d, path) if not os.path.islink(l): m = self.recipe.magic[path] if m and m.name == 'ELF' and 'soname' in m.contents: if os.path.basename(path) == m.contents['soname']: target = m.contents['soname'] + '.something' else: target = m.contents['soname'] self.warn( '%s is not a symlink but probably should be a link to %s', path, target) return # store initial contents sopath = util.joinPaths(os.path.dirname(l), os.readlink(l)) so = util.normpath(sopath) # find final file while os.path.islink(l): l = util.normpath( util.joinPaths(os.path.dirname(l), os.readlink(l))) p = util.joinPaths(d, path) linkpath = l[destlen:] m = self.recipe.magic[linkpath] if m and m.name == 'ELF' and 'soname' in m.contents: if so == linkpath: self.info( '%s is final path, soname is %s;' ' soname usually is symlink to specific implementation', linkpath, m.contents['soname']) soname = util.normpath( util.joinPaths(os.path.dirname(sopath), m.contents['soname'])) s = soname[destlen:] try: os.stat(soname) if not os.path.islink(soname) and s not in self.nonSymlinkWarn: self.nonSymlinkWarn.add(s) self.info( '%s has soname %s; best practice is that the' ' filename that matches the soname is a symlink:' ' soname -> soname.minorversion', s, m.contents['soname']) except OSError: # the missing file case will be fixed up by other policy pass
def do(self): if not self.buildTestSuite: self.recipe.TestSuiteFiles(build=False) return # expand macros in fileMap newFileMap = {} for (buildfile, destfile) in self.fileMap.iteritems(): newFileMap[util.normpath(buildfile % self.macros)] = destfile % self.macros self.fileMap = newFileMap self.builddirfiles = {} self.builddirlinks = {} builddir = self.macros.builddir builddirlen = len(builddir) for root, dirs, files in os.walk(builddir): for file in files: realDir = root[builddirlen:] realPath = os.path.join(realDir, file) if realPath in self.fileMap: continue fullpath = os.path.join(root, file) if os.path.islink(fullpath): # symlink handling: # change to absolute link and add to symlink list contents = os.readlink(fullpath) if contents[0] != '/': contents = util.normpath(os.path.join(root, contents))[builddirlen:] if contents not in self.builddirlinks: self.builddirlinks[contents] = [] self.builddirlinks[contents].append(os.path.join(realDir, file)) else: # add to regular file list if file not in self.builddirfiles: self.builddirfiles[file] = [] self.builddirfiles[file].append(realDir) if self.buildTestSuite: for (buildfile, destfile) in self.fileMap.iteritems(): target = destfile link = util.normpath('%(destdir)s%(thistestdir)s/' % self.macros + buildfile) util.mkdirChain(os.path.dirname(link)) os.symlink(target, link) self.recipe.TestSuiteFiles(build=True) self.recipe.TestSuiteFiles(builddirlinks=self.builddirlinks) policy.Policy.do(self) else: self.recipe.TestSuiteFiles(build=False) return
def doFile(self, path): if hasattr(self.recipe, '_getCapsulePathsForFile'): if self.recipe._getCapsulePathsForFile(path): return d = self.macros.destdir destlen = len(d) l = util.joinPaths(d, path) if not os.path.islink(l): m = self.recipe.magic[path] if m and m.name == 'ELF' and 'soname' in m.contents: if os.path.basename(path) == m.contents['soname']: target = m.contents['soname']+'.something' else: target = m.contents['soname'] self.warn( '%s is not a symlink but probably should be a link to %s', path, target) return # store initial contents sopath = util.joinPaths(os.path.dirname(l), os.readlink(l)) so = util.normpath(sopath) # find final file while os.path.islink(l): l = util.normpath(util.joinPaths(os.path.dirname(l), os.readlink(l))) p = util.joinPaths(d, path) linkpath = l[destlen:] m = self.recipe.magic[linkpath] if m and m.name == 'ELF' and 'soname' in m.contents: if so == linkpath: self.info('%s is final path, soname is %s;' ' soname usually is symlink to specific implementation', linkpath, m.contents['soname']) soname = util.normpath(util.joinPaths( os.path.dirname(sopath), m.contents['soname'])) s = soname[destlen:] try: os.stat(soname) if not os.path.islink(soname) and s not in self.nonSymlinkWarn: self.nonSymlinkWarn.add(s) self.info('%s has soname %s; best practice is that the' ' filename that matches the soname is a symlink:' ' soname -> soname.minorversion', s, m.contents['soname']) except OSError: # the missing file case will be fixed up by other policy pass
def prepareManifestFile(self, package=None): # separate from __init__ for the sake of delayed instantiation # where package is derived from data not available at __init__ time if package is None: package = self.package self.manifestsDir = '%s/%s/_MANIFESTS_' \ % (util.normpath(self.recipe.cfg.buildPath), self.recipe.name) component = None if ':' in package: (package, component) = package.split(':') if package: self.recipe.packages[package] = True i = 0 while True: manifestName = '%s.%d' % (package, i) if manifestName not in self.recipe.manifests: break i += 1 self.name = manifestName self.manifestFile = '%s/%s.manifest' % (self.manifestsDir, manifestName) self.recipe.manifests.add(manifestName) if component: self.recipe.ComponentSpec(component, self.load) if package: self.recipe.PackageSpec(package, self.load)
def recordMove(self, src, dest): destdir = util.normpath(self.macros.destdir) def _removeDestDir(p): p = util.normpath(p) if p[:len(destdir)] == destdir: return p[len(destdir):] else: return p if os.path.isdir(src): # assume move is about to happen baseDir = src postRename = False elif os.path.isdir(dest): # assume move just happened baseDir = dest postRename = True else: # don't walk directories baseDir = None src = _removeDestDir(src) dest = _removeDestDir(dest) self._pathTranslations.append((src, dest)) if baseDir: for base, dirs, files in os.walk(baseDir): for path in dirs + files: if not postRename: fSrc = os.path.join(base, path) fSrc = fSrc.replace(self.macros.destdir, '') fDest = fSrc.replace(src, dest) else: fDest = os.path.join(base, path) fDest = fDest.replace(self.macros.destdir, '') fSrc = fDest.replace(dest, src) self._pathTranslations.append((fSrc, fDest))
def do(self): if hasattr(self.recipe, '_getCapsulePathsForFile'): if self.recipe.getType() == recipe.RECIPE_TYPE_CAPSULE: return macros = self.macros subtrees = self.invariantsubtrees if self.subtrees: subtrees.extend(self.subtrees) for path in subtrees: path = util.normpath(path % macros) fullpath = '/'.join((self.macros.destdir, path)) if not os.path.exists(fullpath): continue # this state can only be reached if SharedLibrary is called with # bad arguments... see CNP-45 mode = os.stat(fullpath)[stat.ST_MODE] if not stat.S_ISDIR(mode): self.error('The subtrees= argument takes directories only;' ' %s is not a directory', path) continue oldfiles = set(os.listdir(fullpath)) bootStrapLdConfig = True ldConfigPath = '%(destdir)s%(essentialsbindir)s/ldconfig'%macros if (not os.path.exists(ldConfigPath)) or self.recipe.isCrossCompiling(): bootStrapLdConfig = False ldConfigPath = '%(essentialsbindir)s/ldconfig'%macros util.execute('%s -n %s' %(ldConfigPath, fullpath)) if not bootStrapLdConfig: db = database.Database(self.recipe.cfg.root, self.recipe.cfg.dbPath) ldConfigTroveName = [ x.getName() for x in db.iterTrovesByPath(ldConfigPath) ] if ldConfigTroveName: ldConfigTroveName = ldConfigTroveName[0] else: ldConfigTroveName = 'glibc:runtime' try: if ldConfigTroveName in self.recipe._getTransitiveBuildRequiresNames(): self.recipe.reportExcessBuildRequires(ldConfigTroveName) else: self.recipe.reportMissingBuildRequires(ldConfigTroveName) except AttributeError: # older Conary that does not have # reportExcessBuildRequires or even the older # reportMissingBuildRequires or # _getTransitiveBuildRequiresNames pass newfiles = set(os.listdir(fullpath)) addedfiles = newfiles - oldfiles removedfiles = oldfiles - newfiles if addedfiles: self.info('ldconfig added the following new files in %s: %s', path, ', '.join(sorted(list(addedfiles)))) if removedfiles: self.warn('ldconfig removed files in %s: %s', path, ', '.join(sorted(list(removedfiles))))
def walkDir(self, ignore, dirname, names): # chop off bit not useful for comparison rootdirlen = len(self.rootdir) path = dirname[rootdirlen:] for name in names: thispath = util.normpath(path + os.sep + name) if self._pathAllowed(thispath): self.doFile(thispath)
def walkDir(self, ignore, dirname, names): # chop off bit not useful for comparison rootdirlen = len(self.rootdir) path=dirname[rootdirlen:] for name in names: thispath = util.normpath(path + os.sep + name) if self._pathAllowed(thispath): self.doFile(thispath)
def doFile(self, path): fullpath = self.macros.destdir + path if os.path.islink(fullpath): return fileName = os.path.basename(path) if fileName in self.builddirfiles: dirName = self.findRightFile(fullpath, fileName, self.builddirfiles[fileName]) if dirName is None: return # if destdir file eq to symlink in builddir if os.path.islink(self.macros.builddir + dirName + os.sep + fileName): return testpath = ('%(destdir)s%(thistestdir)s' + os.sep + dirName + os.sep + fileName) % self.macros util.mkdirChain(os.path.dirname(testpath)) if os.path.islink(testpath): buildpath = ''.join( (self.macros.builddir, dirName, os.sep, fileName)) if self.betterLink(self.macros.destdir + path, testpath, buildpath): util.remove(testpath) os.symlink(util.normpath(path), testpath) else: return else: os.symlink(util.normpath(path), testpath) # we've added a builddir file to the testdir, # see if there was a symlink in the builddir # that pointed to it builddirFile = os.path.join(dirName, fileName) if builddirFile in self.builddirlinks: for path in self.builddirlinks[builddirFile]: linkpath = '%(destdir)s%(thistestdir)s' % self.macros + os.sep + path if not os.path.islink(linkpath): util.mkdirChain(os.path.dirname(linkpath)) os.symlink( '%(thistestdir)s/' % self.macros + dirName + os.sep + fileName, linkpath)
def _testdirlink(self, path, target): fullpath = self._testdirpath(path) tpath = self._ddpath(target) if os.path.islink(fullpath): contents = os.readlink(fullpath) if contents[0] == '/': return contents == tpath else: return util.normpath(os.path.join(os.path.dirname(fullpath), contents)) == tpath return False
def doFile(self, path): fullpath = self.macros.destdir + path if os.path.islink(fullpath): return fileName = os.path.basename(path) if fileName in self.builddirfiles: dirName = self.findRightFile(fullpath, fileName, self.builddirfiles[fileName]) if dirName is None: return # if destdir file eq to symlink in builddir if os.path.islink(self.macros.builddir + dirName + os.sep + fileName): return testpath = ('%(destdir)s%(thistestdir)s' + os.sep + dirName + os.sep + fileName) % self.macros util.mkdirChain(os.path.dirname(testpath)) if os.path.islink(testpath): buildpath = ''.join((self.macros.builddir, dirName, os.sep, fileName)) if self.betterLink(self.macros.destdir + path, testpath, buildpath): util.remove(testpath) os.symlink(util.normpath(path), testpath) else: return else: os.symlink(util.normpath(path), testpath) # we've added a builddir file to the testdir, # see if there was a symlink in the builddir # that pointed to it builddirFile = os.path.join(dirName, fileName) if builddirFile in self.builddirlinks: for path in self.builddirlinks[builddirFile]: linkpath = '%(destdir)s%(thistestdir)s' % self.macros + os.sep + path if not os.path.islink(linkpath): util.mkdirChain(os.path.dirname(linkpath)) os.symlink('%(thistestdir)s/' % self.macros + dirName + os.sep + fileName, linkpath)
def _getBaseDownloadUrl(self): """ Return the base URL relative to which to download images, removing any user/password information, and using http """ # extract the downloadImage base url from the serverUrl configuration parts = list(urllib2.urlparse.urlparse(self.rbuilderUrl)) parts[1] = urllib2.splituser(parts[1])[1] # FIXME: is this whole ../ business a workaround for splitbrain rBO? parts[2] = parts[2] and parts[2] or "/" return "http://%s%s" % (parts[1], util.normpath(parts[2] + "../")[1:])
def _getBaseDownloadUrl(self): ''' Return the base URL relative to which to download images, removing any user/password information, and using http ''' # extract the downloadImage base url from the serverUrl configuration parts = list(urllib2.urlparse.urlparse(self.rbuilderUrl)) parts[1] = urllib2.splituser(parts[1])[1] # FIXME: is this whole ../ business a workaround for splitbrain rBO? parts[2] = parts[2] and parts[2] or '/' return 'http://%s%s' % (parts[1], util.normpath(parts[2] + '../')[1:])
def _gzsymlink(self, dirname, names): for name in names: path = dirname + os.sep + name if os.path.islink(path): # change symlinks to .gz -> .gz contents = os.readlink(path) os.remove(path) if not contents.endswith('.gz'): contents = contents + '.gz' if not path.endswith('.gz'): path = path + '.gz' os.symlink(util.normpath(contents), path)
def recordPaths(self, paths): if not isinstance(paths, (list, tuple, set)): paths = [paths] destdir = util.normpath(self.recipe.macros.destdir) def _removeDestDir(p): p = util.normpath(p) if p[:len(destdir)] == destdir: return p[len(destdir):] else: return p paths = [_removeDestDir(x % self.recipe.macros) for x in paths] self.manifestPaths.update(paths)
def addPluggableRequirements(self, path, fullpath, pkgFiles, macros): mandatoryReqs, optionalReqs = self._parseEggRequires(path, fullpath) filesRequired = [] for req in itertools.chain(mandatoryReqs, optionalReqs): candidatePrefs = [ '%(destdir)s%(libdir)s/python*/', '%(destdir)s%(prefix)s/lib/python*/', '%(destdir)s%(libdir)s/python*/site-packages/', '%(destdir)s%(prefix)s/lib/python*/site-packages/', '%(libdir)s/python*/', '%(prefix)s/lib/python*/', '%(libdir)s/python*/site-packages/', '%(prefix)s/lib/python*/site-packages/', ] candidateFileNames = [(x + req + '*.egg-info/PKG-INFO') % macros \ for x in candidatePrefs] candidateFiles = [fixedglob.glob(x) for x in candidateFileNames] candidateFiles = [x[0] for x in candidateFiles if x] if candidateFiles: filesRequired.append(candidateFiles[0]) else: if req in mandatoryReqs: self.warn('Python egg-info for %s was not found', req) for fileRequired in filesRequired: troveName = None if fileRequired.startswith(macros.destdir): # find requirement in packaging fileRequired = util.normpath(os.path.realpath(fileRequired)) fileRequired = fileRequired[len(util.normpath(macros.destdir) ):] autopkg = self.recipe.autopkg troveName = autopkg.findComponent(fileRequired).getName() else: troveName = self._enforceProvidedPath(fileRequired, fileType='egg-info', unmanagedError=True) if troveName: self._addRequirement(path, troveName, [], pkgFiles, deps.TroveDependencies)
def doFile(self, path): if hasattr(self.recipe, '_getCapsulePathsForFile'): if self.recipe._getCapsulePathsForFile(path): return fullpath = self.macros['destdir'] + path if os.path.islink(fullpath): contents = os.readlink(fullpath) if contents.startswith('/'): pathlist = util.normpath(path).split('/') contentslist = util.normpath(contents).split('/') if pathlist == contentslist: raise policy.PolicyError('Symlink points to itself:' ' %s -> %s' % (path, contents)) while contentslist and pathlist[0] == contentslist[0]: pathlist = pathlist[1:] contentslist = contentslist[1:] os.remove(fullpath) dots = "../" dots *= len(pathlist) - 1 normpath = util.normpath(dots + '/'.join(contentslist)) os.symlink(normpath, fullpath)
def addPluggableRequirements(self, path, fullpath, pkgFiles, macros): mandatoryReqs, optionalReqs = self._parseEggRequires(path, fullpath) filesRequired = [] for req in itertools.chain(mandatoryReqs, optionalReqs): candidatePrefs = [ '%(destdir)s%(libdir)s/python*/', '%(destdir)s%(prefix)s/lib/python*/', '%(destdir)s%(libdir)s/python*/site-packages/', '%(destdir)s%(prefix)s/lib/python*/site-packages/', '%(libdir)s/python*/', '%(prefix)s/lib/python*/', '%(libdir)s/python*/site-packages/', '%(prefix)s/lib/python*/site-packages/', ] candidateFileNames = [(x + req + '*.egg-info/PKG-INFO') % macros \ for x in candidatePrefs] candidateFiles = [fixedglob.glob(x) for x in candidateFileNames] candidateFiles = [x[0] for x in candidateFiles if x] if candidateFiles: filesRequired.append(candidateFiles[0]) else: if req in mandatoryReqs: self.warn('Python egg-info for %s was not found', req) for fileRequired in filesRequired: troveName = None if fileRequired.startswith(macros.destdir): # find requirement in packaging fileRequired = util.normpath(os.path.realpath(fileRequired)) fileRequired = fileRequired[len(util.normpath(macros.destdir)):] autopkg = self.recipe.autopkg troveName = autopkg.findComponent(fileRequired).getName() else: troveName = self._enforceProvidedPath(fileRequired, fileType='egg-info', unmanagedError=True) if troveName: self._addRequirement(path, troveName, [], pkgFiles, deps.TroveDependencies)
def sortMountPoints(mounts): """Sorts mount points by specificity by counting os.path.sep instances.""" mounts = [util.normpath(x) for x in mounts] # sort mounts by specificity, more specific mountpoints first. mounts = sorted(mounts, key = lambda x: x.count(os.path.sep), reverse = True) # / is special, put it at the end if os.path.sep in mounts: mounts.remove(os.path.sep) mounts.append(os.path.sep) return mounts
def doFile(self, path): if hasattr(self.recipe, '_getCapsulePathsForFile'): if self.recipe._getCapsulePathsForFile(path): return fullpath = self.macros['destdir']+path if os.path.islink(fullpath): contents = os.readlink(fullpath) if contents.startswith('/'): pathlist = util.normpath(path).split('/') contentslist = util.normpath(contents).split('/') if pathlist == contentslist: raise policy.PolicyError('Symlink points to itself:' ' %s -> %s' % (path, contents)) while contentslist and pathlist[0] == contentslist[0]: pathlist = pathlist[1:] contentslist = contentslist[1:] os.remove(fullpath) dots = "../" dots *= len(pathlist) - 1 normpath = util.normpath(dots + '/'.join(contentslist)) os.symlink(normpath, fullpath)
def postInit(self): self.runnable = True self.warnedSoNames = set() self.logLines = [] # subscribe to necessary build log entries if hasattr(self.recipe, 'subscribeLogs'): macros = {'cc': re.escape(self.recipe.macros.cc), 'cxx': re.escape(self.recipe.macros.cxx)} regexp = self.regexp % macros self.recipe.subscribeLogs(regexp) self.r = re.compile(regexp) macros = self.recipe.macros cfg = self.recipe.cfg self.libDirs = {'%s%s' %(cfg.root, macros.libdir): macros.libdir, util.normpath('%s/%s'%(cfg.root, macros.lib)): '/%s' %macros.lib} self._initComponentExceptions() else: # disable this policy self.runnable = False
def _removeDestDir(p): p = util.normpath(p) if p[:len(destdir)] == destdir: return p[len(destdir):] else: return p
def getTrovesToDisplay(db, troveSpecs, pathList=[], whatProvidesList=[], exactFlavors=False): """ Finds the given trove and path specifiers, and returns matching (n,v,f) tuples. @param db: database to search @type db: local.database.Database @param troveSpecs: troves to search for @type troveSpecs: list of troveSpecs (n[=v][[f]]) @param pathList: paths which should be linked to some trove in this database. @type pathList: list of strings @param whatProvidesList: deps to search for providers of @type whatProvidesList: list of strings @raises TroveSpecError: Raised if one of the troveSpecs is of an invalid format @note: This function calls database routines which could raise any errors defined in L{dbstore.sqlerrors} @rtype: troveTupleList (list of (name, version, flavor) tuples), and a boolean that stats whether the troves returned should be considered primary (and therefore not compressed ever). """ primary = True if troveSpecs: troveSpecs = [ cmdline.parseTroveSpec(x, allowEmptyName=False) \ for x in troveSpecs ] else: troveSpecs = [] normPathList = [ util.realpath(os.path.abspath(util.normpath(x))) for x in pathList ] troveTups = [] for path, origPath in itertools.izip(normPathList, pathList): if origPath.endswith('/'): allPaths = [ path + '/' + x for x in os.listdir(db.root + path) ] else: allPaths = [ path ] for thisPath in allPaths: for trove in db.iterTrovesByPath(thisPath): troveTups.append((trove.getName(), trove.getVersion(), trove.getFlavor())) if whatProvidesList: results = db.getTrovesWithProvides(whatProvidesList) troveTups.extend(itertools.chain(*results.itervalues())) if not (troveSpecs or pathList or whatProvidesList): troveTups = sorted(db.iterAllTroves()) primary = False else: results = db.findTroves(None, troveSpecs, exactFlavors=exactFlavors) for troveSpec in troveSpecs: troveTups.extend(results.get(troveSpec, [])) return troveTups, primary
def doProcess(self, recipe): """ Invocation instance @param recipe: holds the recipe object, which is used for the macro set and package objects. @return: None @rtype: None """ self.recipe = recipe self.macros = recipe.macros if not self._isSupportedTarget(): return if self.rootdir: self.rootdir = util.normpath(self.rootdir % self.macros) if (hasattr(recipe, '_isDerived') and recipe._isDerived == True and self.processUnmodified is None): # This policy does not handle derived packages return if hasattr(self.__class__, 'preProcess'): self.preProcess() # is runtime check implemented? if hasattr(self.__class__, 'test'): if not self.test(): return # change self.use to be a simple flag self.use = action.checkUse(self.use) # compile the exceptions self.exceptionFilters = [] self.compileFilters(self.invariantexceptions, self.exceptionFilters) if self.exceptions: if not isinstance(self.exceptions, (tuple, list, set)): # turn a plain string into a sequence self.exceptions = (self.exceptions,) self.compileFilters(self.exceptions, self.exceptionFilters, self.unusedFilters['exceptions']) # compile the inclusions self.inclusionFilters = [] if self.invariantinclusions is None: self.compileFilters([], self.inclusionFilters) else: self.compileFilters(self.invariantinclusions, self.inclusionFilters) if not self.inclusions: # an empty list, as opposed to None, means nothing is included if isinstance(self.inclusions, (tuple, list, set)): return else: if not isinstance(self.inclusions, (tuple, list, set)): # turn a plain string into a sequence self.inclusions = (self.inclusions,) self.compileFilters(self.inclusions, self.inclusionFilters, self.unusedFilters['inclusions']) # dispatch if/as appropriate if self.use: self.do() if hasattr(self.__class__, 'postProcess'): self.postProcess()
def _normpath(path): return util.normpath(path).lstrip('/')
def do(self): if hasattr(self.recipe, '_getCapsulePathsForFile'): if self.recipe.getType() == recipe.RECIPE_TYPE_CAPSULE: return macros = self.macros subtrees = self.invariantsubtrees if self.subtrees: subtrees.extend(self.subtrees) for path in subtrees: path = util.normpath(path % macros) fullpath = '/'.join((self.macros.destdir, path)) if not os.path.exists(fullpath): continue # this state can only be reached if SharedLibrary is called with # bad arguments... see CNP-45 mode = os.stat(fullpath)[stat.ST_MODE] if not stat.S_ISDIR(mode): self.error( 'The subtrees= argument takes directories only;' ' %s is not a directory', path) continue oldfiles = set(os.listdir(fullpath)) bootStrapLdConfig = True ldConfigPath = '%(destdir)s%(essentialsbindir)s/ldconfig' % macros if (not os.path.exists(ldConfigPath) ) or self.recipe.isCrossCompiling(): bootStrapLdConfig = False ldConfigPath = '%(essentialsbindir)s/ldconfig' % macros if not os.path.exists(ldConfigPath): self.warn('ldconfig not found') continue util.execute('%s -n %s' % (ldConfigPath, fullpath)) if not bootStrapLdConfig: db = database.Database(self.recipe.cfg.root, self.recipe.cfg.dbPath) ldConfigTroveName = [ x.getName() for x in db.iterTrovesByPath(ldConfigPath) ] if ldConfigTroveName: ldConfigTroveName = ldConfigTroveName[0] else: ldConfigTroveName = 'glibc:runtime' try: if ldConfigTroveName in self.recipe._getTransitiveBuildRequiresNames( ): self.recipe.reportExcessBuildRequires( ldConfigTroveName) else: self.recipe.reportMissingBuildRequires( ldConfigTroveName) except AttributeError: # older Conary that does not have # reportExcessBuildRequires or even the older # reportMissingBuildRequires or # _getTransitiveBuildRequiresNames pass newfiles = set(os.listdir(fullpath)) addedfiles = newfiles - oldfiles removedfiles = oldfiles - newfiles if addedfiles: self.info('ldconfig added the following new files in %s: %s', path, ', '.join(sorted(list(addedfiles)))) if removedfiles: self.warn('ldconfig removed files in %s: %s', path, ', '.join(sorted(list(removedfiles))))
def testLookAsideTest1(self): raise testhelp.SkipTestException("tests shouldn't talk to the internet") self.resetCache() self.resetWork() self.resetRepository() repos = self.openRepository() cfg = self.cfg repCache = lookaside.RepositoryCache(repos, cfg=cfg) origDir = os.getcwd() os.chdir(self.workDir) self.newpkg('test') os.chdir('test') testdir = '/'.join((self.workDir, 'test')) log.setVerbosity(log.INFO) self.logFilter.add() # first, look for a file that does not exist assert(lookaside.findAll(self.cfg, repCache, 'http://example.conary.com/foo', 'test', (testdir,), allowNone=True) is None) # make sure that we got a negative cache entry assert(os.stat('/'.join((self.cacheDir, 'NEGATIVE/test/example.conary.com/foo')))) # now make sure that this works for ftp as well (urllib workaround) # XXX WORKAROUND until FTP works in eng lab #assert(lookaside.findAll(self.cfg, repCache, # 'ftp://download.rpath.com/blah', 'test', (testdir,), # allowNone=True) is None) # make sure that we got a negative cache entry #assert(os.stat('/'.join((self.cacheDir, # 'NEGATIVE/test/download.rpath.com/blah')))) # now we put a file in place f = file(os.sep.join((testdir, 'bar')), 'w') f.write('this is a test') f.close() c = lookaside.findAll(self.cfg, repCache, 'bar', 'test', (testdir,)) # it does not need to cache it; it is known to exist assert(c == os.sep.join((testdir, 'bar'))) # Test httpHeaders: c = util.normpath(lookaside.findAll(self.cfg, repCache, 'http://www.google.com/preferences', 'test', (testdir,), httpHeaders={'Accept-Language': 'es-es'})) assert(c == '/'.join((self.cacheDir, 'test/www.google.com/preferences'))) #open the page and check to see if it's in spanish f = open(c) contents = f.read() f.close() assert 'Preferencias globales' in contents # we need a web page to actually test the cache in operation # we do it a second time to make sure that the cache works for i in (0, 1): c = util.normpath(lookaside.findAll(self.cfg, repCache, 'http://wiki.rpath.com/wiki/Main_Page', 'test', (testdir,))) assert(c == '/'.join((self.cacheDir, 'test/wiki.rpath.com/wiki/Main_Page'))) self.logFilter.remove() self.logFilter.compare( ('+ Trying http://example.conary.com/foo...', # XXX WORKAROUND until FTP works in eng lab #'+ Trying ftp://download.rpath.com/blah...', #'+ Downloading ftp://download.rpath.com/blah...', '+ Trying http://www.google.com/preferences...', '+ Downloading http://www.google.com/preferences...', '+ Trying http://wiki.rpath.com/wiki/Main_Page...', '+ Downloading http://wiki.rpath.com/wiki/Main_Page...')) recipestr = """ class TestLookaside(PackageRecipe): name = 'test' version = '1' clearBuildReqs() def setup(r): r.addSource('bar', dest='/') """ self.writeFile('test.recipe', recipestr) self.addfile('test.recipe') self.addfile('bar', binary = True) self.commit() os.chdir(origDir) # ensure that a localOnly=True lookup in the repository works; # for this, we need a prepped recipeObj for its RepositoryCache # object recipeObj = self.getRecipeObjFromRepos('test', repos) self.logFilter.add() c = lookaside.findAll(self.cfg, recipeObj.laReposCache, 'bar', 'test', (), localOnly=True) self.logFilter.remove() self.logFilter.compare( '+ found bar in repository', ) assert(c == os.sep.join((self.cacheDir, 'test', 'bar')))
def install(self, flags, troveCs): ACTION_RESTORE = 1 ACTION_SKIP = 2 ACTION_CONFLICT = 3 rc = SingleCapsuleOperation.install(self, flags, troveCs) if rc is None: # parent class thinks we should just ignore this troveCs; I'm # not going to argue with it (it's probably because the capsule # hasn't changed return None (oldTrv, trv) = rc trvInfo = troveCs.getNewNameVersionFlavor() oldTrvInfo = troveCs.getOldNameVersionFlavor() hasCapsule = troveCs.hasCapsule() # Updates the fsJob metadata for installing the current trove. # It assumes files are replaced on install, and complains if something # is in the way unless the appropriate flags are set. This is a very # much simplified version of FilesystemJob._singleTrove() which maps # out a complete install strategy for native packages. Note that # we walk all of the files in this trove, not just the new files # or the changed files, because RPM installs all of the files. toRestore = [] changedByPathId = dict((x[0], x) for x in troveCs.getChangedFileList()) # things which aren't change, new, or removed are unchanged unchangedByPathId = (set(x[0] for x in trv.iterFileList()) - set(changedByPathId.iterkeys()) - set(x[0] for x in troveCs.getNewFileList()) - set(troveCs.getOldFileList())) l = [] for oldFileInfo in troveCs.getChangedFileList(): oldFileId, oldVersion = oldTrv.getFile(oldFileInfo[0])[1:3] l.append((oldFileInfo[0], oldFileId, oldVersion)) for unchangedPathId in unchangedByPathId: unchangedFileId, unchangedFileVersion = \ trv.getFile(unchangedPathId)[1:3] l.append((unchangedPathId, unchangedFileId, unchangedFileVersion)) fileObjs = self.db.getFileVersions(l) fileObjsByPathId = dict( [ (x[0], y) for x, y in itertools.izip(l, fileObjs) ] ) for fileInfo in trv.iterFileList(): pathId, path, fileId, version = fileInfo if os.path.dirname(path) in self.netSharedPath: # we do nothing. really. nothing. # # we don't back it up. we don't mark it as removed in # our database. we don't look for conflicts. nothing. continue if pathId in changedByPathId: oldFileId = oldTrv.getFile(pathId)[1] fileChange = self.changeSet.getFileChange(oldFileId, fileId) if (oldFileId == fileId): # only the version number changed; we don't need # to merge anything here fileObj = fileObjsByPathId[pathId] elif fileChange[0] == '\x01': fileObj = fileObjsByPathId[pathId] fileObj.twm(fileChange, fileObj) else: fileObj = files.ThawFile(fileChange, pathId) elif pathId in unchangedByPathId: fileObj = fileObjsByPathId[pathId] else: # if it's not changed and it's not unchanged, it must be new fileStream = self.changeSet.getFileChange(None, fileId) fileObj = files.ThawFile(fileStream, pathId) absolutePath = util.joinPaths(self.root, path) if (fileObj.flags.isCapsuleAddition()): # this was added to the package outside of the RPM; we don't # have any responsibility for it continue elif (trove.conaryContents(hasCapsule, pathId, fileObj) and fileObj.lsTag != 'd'): # this content isn't part of the capsule; remember to put # it back when RPM is done self.preservePath(path) continue s = util.lstat(absolutePath) if not s: # there is nothing in the way, so there is nothing which # concerns us here. Track the file for later. toRestore.append((fileInfo, fileObj)) continue action = ACTION_CONFLICT existingOwners = list( self.db.iterFindPathReferences(path, justPresent = True, withStream = True)) if existingOwners: # Don't complain about files owned by the previous version # of this trove. l = [ x for x in existingOwners if x[0:3] == oldTrvInfo ] if l: existingOwners.remove(l[0]) if not existingOwners: action = ACTION_RESTORE elif stat.S_ISDIR(s.st_mode) and fileObj.lsTag == 'd': # Don't let existing directories stop us from taking over # ownership of the directory action = ACTION_RESTORE elif fileObj.flags.isInitialContents(): # Initial contents files may be restored on top of things # already in the filesystem. They're ghosts or config files # and RPM will get the contents right either way, and we # should remove them either way. action = ACTION_RESTORE if action == ACTION_CONFLICT and existingOwners: if fileId in [ x[4] for x in existingOwners ]: # The files share metadata same. Whatever it looks like on # disk, RPM is going to blow it away with the new one. for info in existingOwners: self.fsJob.sharedFile(info[0], info[1], info[2], info[3]) action = ACTION_RESTORE elif path.startswith('/usr/share/doc/'): # Mirror badness Red Hat patches into RPM for rhel4 # and rhel5 action = ACTION_RESTORE else: existingFiles = [ files.ThawFile(x[5], pathId) for x in existingOwners ] compatibility = [ 1 for x in existingFiles if fileObj.compatibleWith(x) ] if 1 in compatibility: # files can be shared even though the fileId's # are different for info in existingOwners: self.fsJob.sharedFile(info[0], info[1], info[2], info[3]) action = ACTION_RESTORE elif 1 in [ files.rpmFileColorCmp(x, fileObj) for x in existingFiles ]: # rpm file colors and the default rpm setting for # file color policy make elf64 files silently replace # elf32 files. follow that behavior here. # # no, i'm not making this up # # yes, really action = ACTION_SKIP elif (self._checkReplaceManagedFiles(flags, path) or 1 in [ files.rpmFileColorCmp(fileObj, x) for x in existingFiles ]): # The files are different. Bail unless we're supposed # to replace managed files. existingFile = files.FileFromFilesystem(absolutePath, pathId) for info in existingOwners: self.fsJob.userRemoval( fileObj = existingFile, content = filecontents.FromFilesystem(absolutePath), *info[0:4]) action = ACTION_RESTORE else: # it's not up to us to decide if this is a true # conflict; the database layer will do that for # us (see checkPathConflicts) action = ACTION_RESTORE elif flags.replaceUnmanagedFiles: # we don't own it, but it's on disk. RPM will just write over # it and we have the flag saying we're good with that action = ACTION_RESTORE if action == ACTION_RESTORE: # We may proceed, and RPM will replace this file for us. We # need to track that it's being restored to avoid conflicts # with other restorations though. toRestore.append((fileInfo, fileObj)) elif action == ACTION_CONFLICT: # The file exists already, we can't share it, and we're not # allowed to overwrite it. self._error(errors.FileInWayError(util.normpath(path), troveCs.getName(), troveCs.getNewVersion(), troveCs.getNewFlavor())) else: assert(action == ACTION_SKIP) self.preservePath(path) self.fsJob.userRemoval(trv.getName(), trv.getVersion(), trv.getFlavor(), pathId) # toRestore is the list of what is going to be restored. We need to get # the fileObjects which will be created so we can track them in the # filesystem job. This lets the filesystem job look for resolveable # conflicts within this update. We handle newly created files first # and files which have changed (so we have to look up the diff) # a bit later. for fileInfo, fileObj in toRestore: fullPath = util.joinPaths(self.root, path) self.fsJob._restore(fileObj, fullPath, trvInfo, "restoring %s from RPM", restoreFile = False, fileId = fileId)
def addPluggableRequirements(self, path, fullpath, pkgFiles, macros): if hasattr(self.recipe, '_getCapsulePathsForFile'): if self.recipe._getCapsulePathsForFile(path): # since capsules do not convert to relative symlinks, # we cannot depend on getting the realpath. Unless # we resolve that, assume that capsule-provided # dependencies will be sufficient for pkgconfig files. return # parse pkgconfig file variables = {} requirements = set() libDirs = [] libraries = set() variableLineRe = re.compile('^[a-zA-Z0-9]+=') filesRequired = [] pcContents = [x.strip() for x in file(fullpath).readlines()] for pcLine in pcContents: # interpolate variables: assume variables are interpreted # line-by-line while processing pcLineIter = pcLine while True: for var in variables: pcLineIter = pcLineIter.replace(var, variables[var]) if pcLine == pcLineIter: break pcLine = pcLineIter pcLine = pcLineIter if variableLineRe.match(pcLine): key, val = pcLine.split('=', 1) variables['${%s}' %key] = val else: if (pcLine.startswith('Requires') or pcLine.startswith('Lib')) and ':' in pcLine: keyWord, args = pcLine.split(':', 1) # split on ',' and ' ' argList = itertools.chain(*[x.split(',') for x in args.split()]) argList = [x for x in argList if x] if keyWord.startswith('Requires'): versionNext = False for req in argList: if [x for x in '<=>' if x in req]: versionNext = True continue if versionNext: versionNext = False continue requirements.add(req) elif keyWord.startswith('Lib'): for lib in argList: if lib.startswith('-L'): libDirs.append(lib[2:]) elif lib.startswith('-l'): libraries.add(lib[2:]) else: pass # find referenced pkgconfig files and add requirements for req in requirements: candidateFileNames = [ '%(destdir)s%(libdir)s/pkgconfig/'+req+'.pc', '%(destdir)s%(datadir)s/pkgconfig/'+req+'.pc', '%(libdir)s/pkgconfig/'+req+'.pc', '%(datadir)s/pkgconfig/'+req+'.pc', ] candidateFileNames = [ x % macros for x in candidateFileNames ] candidateFiles = [ util.exists(x) for x in candidateFileNames ] if True in candidateFiles: filesRequired.append( (candidateFileNames[candidateFiles.index(True)], 'pkg-config')) else: self.warn('pkg-config file %s.pc not found', req) continue # find referenced library files and add requirements libraryPaths = sorted(list(self.systemLibPaths)) for libDir in libDirs: if libDir not in libraryPaths: libraryPaths.append(libDir) for library in libraries: found = False for libDir in libraryPaths: candidateFileNames = [ macros.destdir+libDir+'/lib'+library+'.so', macros.destdir+libDir+'/lib'+library+'.a', libDir+'/lib'+library+'.so', libDir+'/lib'+library+'.a', ] candidateFiles = [ util.exists(x) for x in candidateFileNames ] if True in candidateFiles: filesRequired.append( (candidateFileNames[candidateFiles.index(True)], 'library')) found = True break if not found: self.warn('library file lib%s not found', library) continue for fileRequired, fileType in filesRequired: if fileRequired.startswith(macros.destdir): # find requirement in packaging fileRequired = util.normpath(os.path.realpath(fileRequired)) fileRequired = fileRequired[len(util.normpath(os.path.realpath(macros.destdir))):] autopkg = self.recipe.autopkg troveName = autopkg.componentMap[fileRequired].name package, component = troveName.split(':', 1) if component in ('devellib', 'lib'): for preferredComponent in ('devel', 'devellib'): develTroveName = ':'.join((package, preferredComponent)) if develTroveName in autopkg.components and autopkg.components[develTroveName]: # found a non-empty :devel compoment troveName = develTroveName break self._addRequirement(path, troveName, [], pkgFiles, deps.TroveDependencies) else: troveName = self._enforceProvidedPath(fileRequired, fileType=fileType, unmanagedError=True) if troveName: self._addRequirement(path, troveName, [], pkgFiles, deps.TroveDependencies)
def testLookAsideTest1(self): raise testhelp.SkipTestException( "tests shouldn't talk to the internet") self.resetCache() self.resetWork() self.resetRepository() repos = self.openRepository() cfg = self.cfg repCache = lookaside.RepositoryCache(repos, cfg=cfg) origDir = os.getcwd() os.chdir(self.workDir) self.newpkg('test') os.chdir('test') testdir = '/'.join((self.workDir, 'test')) log.setVerbosity(log.INFO) self.logFilter.add() # first, look for a file that does not exist assert (lookaside.findAll(self.cfg, repCache, 'http://example.conary.com/foo', 'test', (testdir, ), allowNone=True) is None) # make sure that we got a negative cache entry assert (os.stat('/'.join( (self.cacheDir, 'NEGATIVE/test/example.conary.com/foo')))) # now make sure that this works for ftp as well (urllib workaround) # XXX WORKAROUND until FTP works in eng lab #assert(lookaside.findAll(self.cfg, repCache, # 'ftp://download.rpath.com/blah', 'test', (testdir,), # allowNone=True) is None) # make sure that we got a negative cache entry #assert(os.stat('/'.join((self.cacheDir, # 'NEGATIVE/test/download.rpath.com/blah')))) # now we put a file in place f = file(os.sep.join((testdir, 'bar')), 'w') f.write('this is a test') f.close() c = lookaside.findAll(self.cfg, repCache, 'bar', 'test', (testdir, )) # it does not need to cache it; it is known to exist assert (c == os.sep.join((testdir, 'bar'))) # Test httpHeaders: c = util.normpath( lookaside.findAll(self.cfg, repCache, 'http://www.google.com/preferences', 'test', (testdir, ), httpHeaders={'Accept-Language': 'es-es'})) assert (c == '/'.join( (self.cacheDir, 'test/www.google.com/preferences'))) #open the page and check to see if it's in spanish f = open(c) contents = f.read() f.close() assert 'Preferencias globales' in contents # we need a web page to actually test the cache in operation # we do it a second time to make sure that the cache works for i in (0, 1): c = util.normpath( lookaside.findAll(self.cfg, repCache, 'http://wiki.rpath.com/wiki/Main_Page', 'test', (testdir, ))) assert (c == '/'.join( (self.cacheDir, 'test/wiki.rpath.com/wiki/Main_Page'))) self.logFilter.remove() self.logFilter.compare(( '+ Trying http://example.conary.com/foo...', # XXX WORKAROUND until FTP works in eng lab #'+ Trying ftp://download.rpath.com/blah...', #'+ Downloading ftp://download.rpath.com/blah...', '+ Trying http://www.google.com/preferences...', '+ Downloading http://www.google.com/preferences...', '+ Trying http://wiki.rpath.com/wiki/Main_Page...', '+ Downloading http://wiki.rpath.com/wiki/Main_Page...')) recipestr = """ class TestLookaside(PackageRecipe): name = 'test' version = '1' clearBuildReqs() def setup(r): r.addSource('bar', dest='/') """ self.writeFile('test.recipe', recipestr) self.addfile('test.recipe') self.addfile('bar', binary=True) self.commit() os.chdir(origDir) # ensure that a localOnly=True lookup in the repository works; # for this, we need a prepped recipeObj for its RepositoryCache # object recipeObj = self.getRecipeObjFromRepos('test', repos) self.logFilter.add() c = lookaside.findAll(self.cfg, recipeObj.laReposCache, 'bar', 'test', (), localOnly=True) self.logFilter.remove() self.logFilter.compare('+ found bar in repository', ) assert (c == os.sep.join((self.cacheDir, 'test', 'bar')))
def doProcess(self, recipe): """ Invocation instance @param recipe: holds the recipe object, which is used for the macro set and package objects. @return: None @rtype: None """ self.recipe = recipe self.macros = recipe.macros if not self._isSupportedTarget(): return if self.rootdir: self.rootdir = util.normpath(self.rootdir % self.macros) if (hasattr(recipe, '_isDerived') and recipe._isDerived == True and self.processUnmodified is None): # This policy does not handle derived packages return if hasattr(self.__class__, 'preProcess'): self.preProcess() # is runtime check implemented? if hasattr(self.__class__, 'test'): if not self.test(): return # change self.use to be a simple flag self.use = action.checkUse(self.use) # compile the exceptions self.exceptionFilters = [] self.compileFilters(self.invariantexceptions, self.exceptionFilters) if self.exceptions: if not isinstance(self.exceptions, (tuple, list, set)): # turn a plain string into a sequence self.exceptions = (self.exceptions, ) self.compileFilters(self.exceptions, self.exceptionFilters, self.unusedFilters['exceptions']) # compile the inclusions self.inclusionFilters = [] if self.invariantinclusions is None: self.compileFilters([], self.inclusionFilters) else: self.compileFilters(self.invariantinclusions, self.inclusionFilters) if not self.inclusions: # an empty list, as opposed to None, means nothing is included if isinstance(self.inclusions, (tuple, list, set)): return else: if not isinstance(self.inclusions, (tuple, list, set)): # turn a plain string into a sequence self.inclusions = (self.inclusions, ) self.compileFilters(self.inclusions, self.inclusionFilters, self.unusedFilters['inclusions']) # dispatch if/as appropriate if self.use: self.do() if hasattr(self.__class__, 'postProcess'): self.postProcess()
def do(self): # For the purposes of this policy, the transitive buildRequires # includes suggestions already made for handling shared libraries, # since this policy is explicitly a fallback for the unusual # case of static linking outside of the package being built. transitiveBuildRequires = self.transitiveBuildRequires.union(self.warnedSoNames) cfg = self.recipe.cfg db = database.Database(cfg.root, cfg.dbPath) foundLibNames = set() allPossibleProviders = set() missingBuildRequires = set() self.buildDirLibNames = None destdir = self.recipe.macros.destdir builddir = self.recipe.macros.builddir tooManyChoices = {} noTroveFound = {} noLibraryFound = set() components = self.recipe.autopkg.components pathMap = self.recipe.autopkg.pathMap reqDepSet = deps.DependencySet() sharedLibraryRequires = set() for pkg in components.values(): reqDepSet.union(pkg.requires) for dep in reqDepSet.iterDepsByClass(deps.SonameDependencies): soname = os.path.basename(dep.name).split('.')[0] sharedLibraryRequires.add(soname) if soname.startswith('lib'): sharedLibraryRequires.add(soname[3:]) else: sharedLibraryRequires.add('lib%s' %soname) troveLibraries = set() for path in pathMap.iterkeys(): basename = os.path.basename(path) if basename.startswith('lib') and basename.find('.') >= 0: troveLibraries.add(basename[3:].split('.')[0]) self.recipe.synchronizeLogs() f = file(self.recipe.getSubscribeLogPath()) libRe = re.compile('^-l[a-zA-Z]+$') libDirRe = re.compile('^-L/..*$') def logLineTokens(): for logLine in f: logLine = logLine.strip() if not self.r.match(logLine): continue yield logLine.split() for logLine in self.logLines: yield logLine.split() def pathSetToTroveSet(pathSet): troveSet = set() for path in pathSet: for pathReq in set(trove.getName() for trove in db.iterTrovesByPath(path)): pathReqCandidates = _providesNames(pathReq) # remove any recursive or non-existing buildreqs pathReqCandidates = [x for x in pathReqCandidates if db.hasTroveByName(x)] if not pathReqCandidates: continue allPossibleProviders.update(pathReqCandidates) # only the best option pathReqCandidates = pathReqCandidates[0:1] # now apply exceptions pathReqCandidates = self._removeExceptionsFromList( pathReqCandidates) troveSet.add(pathReqCandidates[0]) return troveSet def buildDirContains(libName): # If we can find this library built somewhere in the # builddir, chances are that the internal library is # what is being linked to in any case. if self.buildDirLibNames is None: # walk builddir once, the first time this is called self.buildDirLibNames = set() for dirpath, dirnames, filenames in os.walk(builddir): for fileName in filenames: if fileName.startswith('lib') and '.' in fileName: self.buildDirLibNames.add(fileName[3:].split('.')[0]) return libName in self.buildDirLibNames for tokens in logLineTokens(): libNames = set(x[2:] for x in tokens if libRe.match(x)) # Add to this set, for this line only, system library dirs, # nothing in destdir or builddir libDirs = self.libDirs.copy() for libDir in set(x[2:].rstrip('/') for x in tokens if libDirRe.match(x) and not x[2:].startswith(destdir) and not x[2:].startswith(builddir)): libDir = util.normpath(libDir) libDirs.setdefault(util.normpath('%s%s' %(cfg.root, libDir)), libDir) libDirs.setdefault(libDir, libDir) for libName in sorted(list(libNames)): if libName not in foundLibNames: if libName in sharedLibraryRequires: foundLibNames.add(libName) continue if libName in troveLibraries: foundLibNames.add(libName) continue if buildDirContains(libName): foundLibNames.add(libName) continue foundLibs = set() for libDirRoot, libDir in libDirs.iteritems(): for ext in ('a', 'so'): # If there is no .a, look for the .so in case # no shared library dependency is found from # packaged files (CNP-132) if util.exists('%s/lib%s.%s' %(libDirRoot, libName, ext)): foundLibs.add('%s/lib%s.%s' %(libDir, libName, ext)) break troveSet = pathSetToTroveSet(foundLibs) if len(troveSet) == 1: # found just one, we can confidently recommend it recommended = list(troveSet)[0] if recommended not in transitiveBuildRequires: self.info("Add '%s' to buildRequires for -l%s (%s)", recommended, libName, ', '.join(sorted(list(foundLibs)))) missingBuildRequires.add(recommended) foundLibNames.add(libName) elif len(troveSet): # found more, we might need to recommend a choice tooManyChoices.setdefault(libName, [ ' '.join(sorted(list(foundLibs))), "', '".join(sorted(list(troveSet)))]) elif foundLibs: # found files on system, but no troves providing them noTroveFound.setdefault(libName, ' '.join(sorted(list(foundLibs)))) else: # note that this does not prevent us from # *looking* again, because the next time # there might be a useful -L in the link line noLibraryFound.add(libName) if tooManyChoices: for libName in sorted(list(tooManyChoices.keys())): if libName not in foundLibNames: # Found multiple choices for libName, and never came # up with a better recommendation, so recommend a choice. # Note: perhaps someday this can become an error # when we have a better sense of how frequently # it is wrong... foundLibNames.add(libName) foundLibs, troveSet = tooManyChoices[libName] self.warn('Multiple troves match files %s for -l%s:' ' choose one of the following entries' " for buildRequires: '%s'", foundLibs, libName, troveSet) if noTroveFound: for libName in sorted(list(noTroveFound.keys())): if libName not in foundLibNames: # Never found any trove containing these libraries, # not even a file in the builddir foundLibNames.add(libName) foundLibs = noTroveFound[libName] self.info('No trove found matching any of files' ' %s for -l%s:' ' possible missing buildRequires', foundLibs, libName) if noLibraryFound: for libName in sorted(list(noLibraryFound)): if libName not in foundLibNames: # Note: perhaps someday this can become an error # when we have a better sense of how frequently # it is wrong... self.info('No files found matching -l%s:' ' possible missing buildRequires', libName) if missingBuildRequires: self.talk('add to buildRequires: %s', str(sorted(list(missingBuildRequires)))) reportMissingBuildRequires(self.recipe, missingBuildRequires) if allPossibleProviders: reportFoundBuildRequires(self.recipe, allPossibleProviders) f.close()
def install(self, flags, troveCs): ACTION_RESTORE = 1 ACTION_SKIP = 2 ACTION_CONFLICT = 3 rc = SingleCapsuleOperation.install(self, flags, troveCs) if rc is None: # parent class thinks we should just ignore this troveCs; I'm # not going to argue with it (it's probably because the capsule # hasn't changed return None (oldTrv, trv) = rc trvInfo = troveCs.getNewNameVersionFlavor() oldTrvInfo = troveCs.getOldNameVersionFlavor() hasCapsule = troveCs.hasCapsule() # Updates the fsJob metadata for installing the current trove. # It assumes files are replaced on install, and complains if something # is in the way unless the appropriate flags are set. This is a very # much simplified version of FilesystemJob._singleTrove() which maps # out a complete install strategy for native packages. Note that # we walk all of the files in this trove, not just the new files # or the changed files, because RPM installs all of the files. toRestore = [] changedByPathId = dict((x[0], x) for x in troveCs.getChangedFileList()) # things which aren't change, new, or removed are unchanged unchangedByPathId = (set(x[0] for x in trv.iterFileList()) - set(changedByPathId.iterkeys()) - set(x[0] for x in troveCs.getNewFileList()) - set(troveCs.getOldFileList())) l = [] for oldFileInfo in troveCs.getChangedFileList(): oldFileId, oldVersion = oldTrv.getFile(oldFileInfo[0])[1:3] l.append((oldFileInfo[0], oldFileId, oldVersion)) for unchangedPathId in unchangedByPathId: unchangedFileId, unchangedFileVersion = \ trv.getFile(unchangedPathId)[1:3] l.append((unchangedPathId, unchangedFileId, unchangedFileVersion)) fileObjs = self.db.getFileVersions(l) fileObjsByPathId = dict([(x[0], y) for x, y in itertools.izip(l, fileObjs)]) for fileInfo in trv.iterFileList(): pathId, path, fileId, version = fileInfo if os.path.dirname(path) in self.netSharedPath: # we do nothing. really. nothing. # # we don't back it up. we don't mark it as removed in # our database. we don't look for conflicts. nothing. continue if pathId in changedByPathId: oldFileId = oldTrv.getFile(pathId)[1] fileChange = self.changeSet.getFileChange(oldFileId, fileId) if (oldFileId == fileId): # only the version number changed; we don't need # to merge anything here fileObj = fileObjsByPathId[pathId] elif fileChange[0] == '\x01': fileObj = fileObjsByPathId[pathId] fileObj.twm(fileChange, fileObj) else: fileObj = files.ThawFile(fileChange, pathId) elif pathId in unchangedByPathId: fileObj = fileObjsByPathId[pathId] else: # if it's not changed and it's not unchanged, it must be new fileStream = self.changeSet.getFileChange(None, fileId) fileObj = files.ThawFile(fileStream, pathId) absolutePath = util.joinPaths(self.root, path) if (fileObj.flags.isCapsuleAddition()): # this was added to the package outside of the RPM; we don't # have any responsibility for it continue elif (trove.conaryContents(hasCapsule, pathId, fileObj) and fileObj.lsTag != 'd'): # this content isn't part of the capsule; remember to put # it back when RPM is done self.preservePath(path, unlink=True) continue s = util.lstat(absolutePath) if not s: # there is nothing in the way, so there is nothing which # concerns us here. Track the file for later. toRestore.append((fileInfo, fileObj)) continue action = ACTION_CONFLICT existingOwners = list( self.db.iterFindPathReferences(path, justPresent=True, withStream=True)) if existingOwners: # Don't complain about files owned by the previous version # of this trove. l = [x for x in existingOwners if x[0:3] == oldTrvInfo] if l: existingOwners.remove(l[0]) if not existingOwners: action = ACTION_RESTORE elif stat.S_ISDIR(s.st_mode) and fileObj.lsTag == 'd': # Don't let existing directories stop us from taking over # ownership of the directory action = ACTION_RESTORE elif fileObj.flags.isInitialContents(): # Initial contents files may be restored on top of things # already in the filesystem. They're ghosts or config files # and RPM will get the contents right either way, and we # should remove them either way. action = ACTION_RESTORE if action == ACTION_CONFLICT and not existingOwners: # Check for "conflicts" that might just be a view across a # symlink. if self.fsJob.findAliasedRemovals(absolutePath): action = ACTION_RESTORE if action == ACTION_CONFLICT and existingOwners: if fileId in [x[4] for x in existingOwners]: # The files share metadata same. Whatever it looks like on # disk, RPM is going to blow it away with the new one. for info in existingOwners: self.fsJob.sharedFile(info[0], info[1], info[2], info[3]) action = ACTION_RESTORE elif path.startswith('/usr/share/doc/'): # Mirror badness Red Hat patches into RPM for rhel4 # and rhel5 action = ACTION_RESTORE else: existingFiles = [ files.ThawFile(x[5], pathId) for x in existingOwners ] compatibility = [ 1 for x in existingFiles if fileObj.compatibleWith(x) ] if 1 in compatibility: # files can be shared even though the fileId's # are different for info in existingOwners: self.fsJob.sharedFile(info[0], info[1], info[2], info[3]) action = ACTION_RESTORE elif 1 in [ files.rpmFileColorCmp(x, fileObj) for x in existingFiles ]: # rpm file colors and the default rpm setting for # file color policy make elf64 files silently replace # elf32 files. follow that behavior here. # # no, i'm not making this up # # yes, really action = ACTION_SKIP elif (self._checkReplaceManagedFiles(flags, path) or 1 in [ files.rpmFileColorCmp(fileObj, x) for x in existingFiles ]): # The files are different. Bail unless we're supposed # to replace managed files. existingFile = files.FileFromFilesystem( absolutePath, pathId) for info in existingOwners: self.fsJob.userRemoval( fileObj=existingFile, content=filecontents.FromFilesystem( absolutePath), *info[0:4]) action = ACTION_RESTORE else: # it's not up to us to decide if this is a true # conflict; the database layer will do that for # us (see checkPathConflicts) action = ACTION_RESTORE elif flags.replaceUnmanagedFiles: # we don't own it, but it's on disk. RPM will just write over # it and we have the flag saying we're good with that action = ACTION_RESTORE if action == ACTION_RESTORE: # We may proceed, and RPM will replace this file for us. We # need to track that it's being restored to avoid conflicts # with other restorations though. toRestore.append((fileInfo, fileObj)) elif action == ACTION_CONFLICT: # The file exists already, we can't share it, and we're not # allowed to overwrite it. self._error( errors.FileInWayError(util.normpath(path), troveCs.getName(), troveCs.getNewVersion(), troveCs.getNewFlavor())) else: assert (action == ACTION_SKIP) self.preservePath(path, unlink=False) self.fsJob.userRemoval(trv.getName(), trv.getVersion(), trv.getFlavor(), pathId) # toRestore is the list of what is going to be restored. We need to get # the fileObjects which will be created so we can track them in the # filesystem job. This lets the filesystem job look for resolveable # conflicts within this update. We handle newly created files first # and files which have changed (so we have to look up the diff) # a bit later. for fileInfo, fileObj in toRestore: fullPath = util.joinPaths(self.root, path) self.fsJob._restore(fileObj, fullPath, trvInfo, "restoring %s from RPM", restoreFile=False, fileId=fileId)
def getTrovesToDisplay(db, troveSpecs, pathList=[], whatProvidesList=[], exactFlavors=False): """ Finds the given trove and path specifiers, and returns matching (n,v,f) tuples. @param db: database to search @type db: local.database.Database @param troveSpecs: troves to search for @type troveSpecs: list of troveSpecs (n[=v][[f]]) @param pathList: paths which should be linked to some trove in this database. @type pathList: list of strings @param whatProvidesList: deps to search for providers of @type whatProvidesList: list of strings @raises TroveSpecError: Raised if one of the troveSpecs is of an invalid format @note: This function calls database routines which could raise any errors defined in L{dbstore.sqlerrors} @rtype: troveTupleList (list of (name, version, flavor) tuples), and a boolean that stats whether the troves returned should be considered primary (and therefore not compressed ever). """ primary = True if troveSpecs: troveSpecs = [ cmdline.parseTroveSpec(x, allowEmptyName=False) \ for x in troveSpecs ] else: troveSpecs = [] normPathList = [ util.realpath(os.path.abspath(util.normpath(x))) for x in pathList ] troveTups = [] for path, origPath in itertools.izip(normPathList, pathList): if origPath.endswith('/'): allPaths = [path + '/' + x for x in os.listdir(db.root + path)] else: allPaths = [path] for thisPath in allPaths: for trove in db.iterTrovesByPath(thisPath): troveTups.append( (trove.getName(), trove.getVersion(), trove.getFlavor())) if whatProvidesList: results = db.getTrovesWithProvides(whatProvidesList) troveTups.extend(itertools.chain(*results.itervalues())) if not (troveSpecs or pathList or whatProvidesList): troveTups = sorted(db.iterAllTroves()) primary = False else: results = db.findTroves(None, troveSpecs, exactFlavors=exactFlavors) for troveSpec in troveSpecs: troveTups.extend(results.get(troveSpec, [])) return troveTups, primary
def addPluggableRequirements(self, path, fullpath, pkgFiles, macros): if hasattr(self.recipe, '_getCapsulePathsForFile'): if self.recipe._getCapsulePathsForFile(path): # since capsules do not convert to relative symlinks, # we cannot depend on getting the realpath. Unless # we resolve that, assume that capsule-provided # dependencies will be sufficient for pkgconfig files. return # parse pkgconfig file variables = {} requirements = set() libDirs = [] libraries = set() variableLineRe = re.compile('^[a-zA-Z0-9]+=') filesRequired = [] pcContents = [x.strip() for x in file(fullpath).readlines()] for pcLine in pcContents: # interpolate variables: assume variables are interpreted # line-by-line while processing pcLineIter = pcLine while True: for var in variables: pcLineIter = pcLineIter.replace(var, variables[var]) if pcLine == pcLineIter: break pcLine = pcLineIter pcLine = pcLineIter if variableLineRe.match(pcLine): key, val = pcLine.split('=', 1) variables['${%s}' % key] = val else: if (pcLine.startswith('Requires') or pcLine.startswith('Lib')) and ':' in pcLine: keyWord, args = pcLine.split(':', 1) # split on ',' and ' ' argList = itertools.chain( *[x.split(',') for x in args.split()]) argList = [x for x in argList if x] if keyWord.startswith('Requires'): versionNext = False for req in argList: if [x for x in '<=>' if x in req]: versionNext = True continue if versionNext: versionNext = False continue requirements.add(req) elif keyWord.startswith('Lib'): for lib in argList: if lib.startswith('-L'): libDirs.append(lib[2:]) elif lib.startswith('-l'): libraries.add(lib[2:]) else: pass # find referenced pkgconfig files and add requirements for req in requirements: candidateFileNames = [ '%(destdir)s%(libdir)s/pkgconfig/' + req + '.pc', '%(destdir)s%(datadir)s/pkgconfig/' + req + '.pc', '%(libdir)s/pkgconfig/' + req + '.pc', '%(datadir)s/pkgconfig/' + req + '.pc', ] candidateFileNames = [x % macros for x in candidateFileNames] candidateFiles = [util.exists(x) for x in candidateFileNames] if True in candidateFiles: filesRequired.append( (candidateFileNames[candidateFiles.index(True)], 'pkg-config')) else: self.warn('pkg-config file %s.pc not found', req) continue # find referenced library files and add requirements libraryPaths = sorted(list(self.systemLibPaths)) for libDir in libDirs: if libDir not in libraryPaths: libraryPaths.append(libDir) for library in libraries: found = False for libDir in libraryPaths: candidateFileNames = [ macros.destdir + libDir + '/lib' + library + '.so', macros.destdir + libDir + '/lib' + library + '.a', libDir + '/lib' + library + '.so', libDir + '/lib' + library + '.a', ] candidateFiles = [util.exists(x) for x in candidateFileNames] if True in candidateFiles: filesRequired.append( (candidateFileNames[candidateFiles.index(True)], 'library')) found = True break if not found: self.warn('library file lib%s not found', library) continue for fileRequired, fileType in filesRequired: if fileRequired.startswith(macros.destdir): # find requirement in packaging fileRequired = util.normpath(os.path.realpath(fileRequired)) fileRequired = fileRequired[ len(util.normpath(os.path.realpath(macros.destdir))):] autopkg = self.recipe.autopkg troveName = autopkg.componentMap[fileRequired].name package, component = troveName.split(':', 1) if component in ('devellib', 'lib'): for preferredComponent in ('devel', 'devellib'): develTroveName = ':'.join( (package, preferredComponent)) if develTroveName in autopkg.components and autopkg.components[ develTroveName]: # found a non-empty :devel compoment troveName = develTroveName break self._addRequirement(path, troveName, [], pkgFiles, deps.TroveDependencies) else: troveName = self._enforceProvidedPath(fileRequired, fileType=fileType, unmanagedError=True) if troveName: self._addRequirement(path, troveName, [], pkgFiles, deps.TroveDependencies)