Exemplo n.º 1
0
    def handleCheck(self, startGroups, stopGroups, lines, fullpath):

        if stopGroups is None:
            # we lost sync, don't start guessing because we care about
            # the result of the check
            return
        sought = startGroups[0]
        success = self.parseSuccess(stopGroups[0])
        includeDirs = [ '%(includedir)s/' %self.macros]
        root = self.recipe.cfg.root

        if success:
            if self.headerRe.match(sought):
                # look for header files
                for tokens in (x.split() for x in lines):
                    for token in tokens:
                        if token.startswith('-I/') and len(token) > 3:
                            includeDirs.append(token[2:])
                for dirName in includeDirs:
                    seekPath = util.normpath('%s/%s' %(dirName, sought))
                    if util.exists('%s%s' %(root, seekPath)):
                        self.foundPaths.add(seekPath)
                        break

            libName = self.libRe.match(sought)
            if libName:
                libName = libName.group(0)
                # Take advantage of the fact that the actual test will
                # include running the compiler with the library in the
                # link line in such a way that the
                # EnforceStaticLibBuildRequirements policy knows how
                # to understand it.
                # EnforceStaticLibBuildRequirements does not handle the
                # leading "configure:01234: " portion of the output,
                # so give it every line that has further content and
                # let it find the lines that it cares about
                logLines = (x.split(': ', 1) for x in lines)
                logLines = (x[1] for x in logLines if len(x) > 1)
                self.recipe.EnforceStaticLibBuildRequirements(logLines=logLines)

            candidate = None
            if sought.startswith('/'):
                candidate = sought
            elif type(success) is str and success.startswith('/'):
                candidate = success
            if candidate:
                # configure:2345: checking for /bin/sought
                # configure:6543: result: yes
                # configure:4441: checking for egrep
                # configure:4519: result: /bin/grep -E
                # configure:4535: checking for ld used by gcc
                # configure:4602: result: /usr/bin/ld
                seekPath = candidate.split()[0]
                if util.exists(util.normpath('%s%s' %(root, seekPath))):
                    self.foundPaths.update(set(
                        self.greylistFilter(set((seekPath,)), fullpath)))
Exemplo n.º 2
0
    def do(self):
        if not self.buildTestSuite:
            self.recipe.TestSuiteFiles(build=False)
            return

        # expand macros in fileMap
        newFileMap = {}
        for (buildfile, destfile) in self.fileMap.iteritems():
            newFileMap[util.normpath(buildfile %
                                     self.macros)] = destfile % self.macros
        self.fileMap = newFileMap

        self.builddirfiles = {}
        self.builddirlinks = {}
        builddir = self.macros.builddir
        builddirlen = len(builddir)
        for root, dirs, files in os.walk(builddir):
            for file in files:
                realDir = root[builddirlen:]
                realPath = os.path.join(realDir, file)
                if realPath in self.fileMap:
                    continue
                fullpath = os.path.join(root, file)

                if os.path.islink(fullpath):
                    # symlink handling:
                    # change to absolute link and add to symlink list
                    contents = os.readlink(fullpath)
                    if contents[0] != '/':
                        contents = util.normpath(os.path.join(
                            root, contents))[builddirlen:]
                    if contents not in self.builddirlinks:
                        self.builddirlinks[contents] = []
                    self.builddirlinks[contents].append(
                        os.path.join(realDir, file))
                else:
                    # add to regular file list
                    if file not in self.builddirfiles:
                        self.builddirfiles[file] = []
                    self.builddirfiles[file].append(realDir)

        if self.buildTestSuite:
            for (buildfile, destfile) in self.fileMap.iteritems():
                target = destfile
                link = util.normpath('%(destdir)s%(thistestdir)s/' %
                                     self.macros + buildfile)
                util.mkdirChain(os.path.dirname(link))
                os.symlink(target, link)

            self.recipe.TestSuiteFiles(build=True)
            self.recipe.TestSuiteFiles(builddirlinks=self.builddirlinks)
            policy.Policy.do(self)
        else:
            self.recipe.TestSuiteFiles(build=False)
            return
Exemplo n.º 3
0
    def doFile(self, path):
        if hasattr(self.recipe, '_getCapsulePathsForFile'):
            if self.recipe._getCapsulePathsForFile(path):
                return

        d = self.macros.destdir
        destlen = len(d)
        l = util.joinPaths(d, path)
        if not os.path.islink(l):
            m = self.recipe.magic[path]
            if m and m.name == 'ELF' and 'soname' in m.contents:
                if os.path.basename(path) == m.contents['soname']:
                    target = m.contents['soname'] + '.something'
                else:
                    target = m.contents['soname']
                self.warn(
                    '%s is not a symlink but probably should be a link to %s',
                    path, target)
            return

        # store initial contents
        sopath = util.joinPaths(os.path.dirname(l), os.readlink(l))
        so = util.normpath(sopath)
        # find final file
        while os.path.islink(l):
            l = util.normpath(
                util.joinPaths(os.path.dirname(l), os.readlink(l)))

        p = util.joinPaths(d, path)
        linkpath = l[destlen:]
        m = self.recipe.magic[linkpath]

        if m and m.name == 'ELF' and 'soname' in m.contents:
            if so == linkpath:
                self.info(
                    '%s is final path, soname is %s;'
                    ' soname usually is symlink to specific implementation',
                    linkpath, m.contents['soname'])
            soname = util.normpath(
                util.joinPaths(os.path.dirname(sopath), m.contents['soname']))
            s = soname[destlen:]
            try:
                os.stat(soname)
                if not os.path.islink(soname) and s not in self.nonSymlinkWarn:
                    self.nonSymlinkWarn.add(s)
                    self.info(
                        '%s has soname %s; best practice is that the'
                        ' filename that matches the soname is a symlink:'
                        ' soname -> soname.minorversion', s,
                        m.contents['soname'])
            except OSError:
                # the missing file case will be fixed up by other policy
                pass
Exemplo n.º 4
0
    def do(self):
        if not self.buildTestSuite:
            self.recipe.TestSuiteFiles(build=False)
            return

        # expand macros in fileMap
        newFileMap = {}
        for (buildfile, destfile) in self.fileMap.iteritems():
            newFileMap[util.normpath(buildfile % self.macros)] = destfile % self.macros
        self.fileMap = newFileMap

        self.builddirfiles = {}
        self.builddirlinks = {}
        builddir = self.macros.builddir
        builddirlen = len(builddir)
        for root, dirs, files in os.walk(builddir):
            for file in files:
                realDir = root[builddirlen:]
                realPath = os.path.join(realDir, file)
                if realPath in self.fileMap:
                    continue
                fullpath = os.path.join(root, file)

                if os.path.islink(fullpath):
                    # symlink handling:
                    # change to absolute link and add to symlink list
                    contents = os.readlink(fullpath)
                    if contents[0] != '/':
                        contents = util.normpath(os.path.join(root, contents))[builddirlen:]
                    if contents not in self.builddirlinks:
                        self.builddirlinks[contents] = []
                    self.builddirlinks[contents].append(os.path.join(realDir, file))
                else:
                    # add to regular file list
                    if file not in self.builddirfiles:
                        self.builddirfiles[file] = []
                    self.builddirfiles[file].append(realDir)


        if self.buildTestSuite:
            for (buildfile, destfile) in self.fileMap.iteritems():
                target = destfile
                link = util.normpath('%(destdir)s%(thistestdir)s/' % self.macros + buildfile)
                util.mkdirChain(os.path.dirname(link))
                os.symlink(target, link)

            self.recipe.TestSuiteFiles(build=True)
            self.recipe.TestSuiteFiles(builddirlinks=self.builddirlinks)
            policy.Policy.do(self)
        else:
            self.recipe.TestSuiteFiles(build=False)
            return
Exemplo n.º 5
0
    def doFile(self, path):
        if hasattr(self.recipe, '_getCapsulePathsForFile'):
            if self.recipe._getCapsulePathsForFile(path):
                return

        d = self.macros.destdir
        destlen = len(d)
        l = util.joinPaths(d, path)
        if not os.path.islink(l):
            m = self.recipe.magic[path]
            if m and m.name == 'ELF' and 'soname' in m.contents:
                if os.path.basename(path) == m.contents['soname']:
                    target = m.contents['soname']+'.something'
                else:
                    target = m.contents['soname']
                self.warn(
                    '%s is not a symlink but probably should be a link to %s',
                    path, target)
            return

        # store initial contents
        sopath = util.joinPaths(os.path.dirname(l), os.readlink(l))
        so = util.normpath(sopath)
        # find final file
        while os.path.islink(l):
            l = util.normpath(util.joinPaths(os.path.dirname(l),
                                             os.readlink(l)))

        p = util.joinPaths(d, path)
        linkpath = l[destlen:]
        m = self.recipe.magic[linkpath]

        if m and m.name == 'ELF' and 'soname' in m.contents:
            if so == linkpath:
                self.info('%s is final path, soname is %s;'
                    ' soname usually is symlink to specific implementation',
                    linkpath, m.contents['soname'])
            soname = util.normpath(util.joinPaths(
                        os.path.dirname(sopath), m.contents['soname']))
            s = soname[destlen:]
            try:
                os.stat(soname)
                if not os.path.islink(soname) and s not in self.nonSymlinkWarn:
                    self.nonSymlinkWarn.add(s)
                    self.info('%s has soname %s; best practice is that the'
                              ' filename that matches the soname is a symlink:'
                              ' soname -> soname.minorversion',
                              s, m.contents['soname'])
            except OSError:
                # the missing file case will be fixed up by other policy
                pass
Exemplo n.º 6
0
    def prepareManifestFile(self, package=None):
        # separate from __init__ for the sake of delayed instantiation
        # where package is derived from data not available at __init__ time
        if package is None:
            package = self.package

        self.manifestsDir = '%s/%s/_MANIFESTS_' \
            % (util.normpath(self.recipe.cfg.buildPath), self.recipe.name)

        component = None

        if ':' in package:
            (package, component) = package.split(':')
        if package:
            self.recipe.packages[package] = True

        i = 0
        while True:
            manifestName = '%s.%d' % (package, i)
            if manifestName not in self.recipe.manifests:
                break
            i += 1

        self.name = manifestName
        self.manifestFile = '%s/%s.manifest' % (self.manifestsDir,
                                                manifestName)
        self.recipe.manifests.add(manifestName)

        if component:
            self.recipe.ComponentSpec(component, self.load)
        if package:
            self.recipe.PackageSpec(package, self.load)
Exemplo n.º 7
0
 def recordMove(self, src, dest):
     destdir = util.normpath(self.macros.destdir)
     def _removeDestDir(p):
         p = util.normpath(p)
         if p[:len(destdir)] == destdir:
             return p[len(destdir):]
         else:
             return p
     if os.path.isdir(src):
         # assume move is about to happen
         baseDir = src
         postRename = False
     elif os.path.isdir(dest):
         # assume move just happened
         baseDir = dest
         postRename = True
     else:
         # don't walk directories
         baseDir = None
     src = _removeDestDir(src)
     dest = _removeDestDir(dest)
     self._pathTranslations.append((src, dest))
     if baseDir:
         for base, dirs, files in os.walk(baseDir):
             for path in dirs + files:
                 if not postRename:
                     fSrc = os.path.join(base, path)
                     fSrc = fSrc.replace(self.macros.destdir, '')
                     fDest = fSrc.replace(src, dest)
                 else:
                     fDest = os.path.join(base, path)
                     fDest = fDest.replace(self.macros.destdir, '')
                     fSrc = fDest.replace(dest, src)
                 self._pathTranslations.append((fSrc, fDest))
Exemplo n.º 8
0
    def prepareManifestFile(self, package=None):
        # separate from __init__ for the sake of delayed instantiation
        # where package is derived from data not available at __init__ time
        if package is None:
            package = self.package

        self.manifestsDir = '%s/%s/_MANIFESTS_' \
            % (util.normpath(self.recipe.cfg.buildPath), self.recipe.name)

        component = None

        if ':' in package:
            (package, component) = package.split(':')
        if package:
            self.recipe.packages[package] = True

        i = 0
        while True:
            manifestName = '%s.%d' % (package, i)
            if manifestName not in self.recipe.manifests:
                break
            i += 1

        self.name = manifestName
        self.manifestFile = '%s/%s.manifest' % (self.manifestsDir, manifestName)
        self.recipe.manifests.add(manifestName)

        if component:
            self.recipe.ComponentSpec(component, self.load)
        if package:
            self.recipe.PackageSpec(package, self.load)
Exemplo n.º 9
0
    def do(self):
        if hasattr(self.recipe, '_getCapsulePathsForFile'):
            if self.recipe.getType() == recipe.RECIPE_TYPE_CAPSULE:
                return
        macros = self.macros
        subtrees = self.invariantsubtrees
        if self.subtrees:
            subtrees.extend(self.subtrees)
        for path in subtrees:
            path = util.normpath(path % macros)
            fullpath = '/'.join((self.macros.destdir, path))
            if not os.path.exists(fullpath):
                continue
            # this state can only be reached if SharedLibrary is called with
            # bad arguments... see CNP-45
            mode = os.stat(fullpath)[stat.ST_MODE]
            if not stat.S_ISDIR(mode):
                self.error('The subtrees= argument takes directories only;'
                           ' %s is not a directory', path)
                continue
            oldfiles = set(os.listdir(fullpath))
            bootStrapLdConfig = True
            ldConfigPath = '%(destdir)s%(essentialsbindir)s/ldconfig'%macros
            if (not os.path.exists(ldConfigPath)) or self.recipe.isCrossCompiling():
                bootStrapLdConfig = False
                ldConfigPath = '%(essentialsbindir)s/ldconfig'%macros
            util.execute('%s -n %s' %(ldConfigPath, fullpath))

            if not bootStrapLdConfig:
                db = database.Database(self.recipe.cfg.root,
                                       self.recipe.cfg.dbPath)
                ldConfigTroveName = [ x.getName() for x in
                                      db.iterTrovesByPath(ldConfigPath) ]
                if ldConfigTroveName:
                    ldConfigTroveName = ldConfigTroveName[0]
                else:
                    ldConfigTroveName = 'glibc:runtime'

                try:
                    if ldConfigTroveName in self.recipe._getTransitiveBuildRequiresNames():
                        self.recipe.reportExcessBuildRequires(ldConfigTroveName)
                    else:
                        self.recipe.reportMissingBuildRequires(ldConfigTroveName)
                except AttributeError:
                    # older Conary that does not have
                    # reportExcessBuildRequires or even the older
                    # reportMissingBuildRequires or
                    # _getTransitiveBuildRequiresNames
                    pass

            newfiles = set(os.listdir(fullpath))
            addedfiles = newfiles - oldfiles
            removedfiles = oldfiles - newfiles
            if addedfiles:
                self.info('ldconfig added the following new files in %s: %s',
                          path, ', '.join(sorted(list(addedfiles))))
            if removedfiles:
                self.warn('ldconfig removed files in %s: %s', path,
                          ', '.join(sorted(list(removedfiles))))
Exemplo n.º 10
0
 def walkDir(self, ignore, dirname, names):
     # chop off bit not useful for comparison
     rootdirlen = len(self.rootdir)
     path = dirname[rootdirlen:]
     for name in names:
         thispath = util.normpath(path + os.sep + name)
         if self._pathAllowed(thispath):
             self.doFile(thispath)
Exemplo n.º 11
0
 def walkDir(self, ignore, dirname, names):
     # chop off bit not useful for comparison
     rootdirlen = len(self.rootdir)
     path=dirname[rootdirlen:]
     for name in names:
        thispath = util.normpath(path + os.sep + name)
        if self._pathAllowed(thispath):
            self.doFile(thispath)
Exemplo n.º 12
0
    def doFile(self, path):
        fullpath = self.macros.destdir + path
        if os.path.islink(fullpath):
            return

        fileName = os.path.basename(path)
        if fileName in self.builddirfiles:
            dirName = self.findRightFile(fullpath, fileName,
                                         self.builddirfiles[fileName])
            if dirName is None:
                return

            # if destdir file eq to symlink in builddir
            if os.path.islink(self.macros.builddir + dirName + os.sep +
                              fileName):
                return

            testpath = ('%(destdir)s%(thistestdir)s' + os.sep + dirName +
                        os.sep + fileName) % self.macros
            util.mkdirChain(os.path.dirname(testpath))

            if os.path.islink(testpath):
                buildpath = ''.join(
                    (self.macros.builddir, dirName, os.sep, fileName))
                if self.betterLink(self.macros.destdir + path, testpath,
                                   buildpath):
                    util.remove(testpath)
                    os.symlink(util.normpath(path), testpath)
                else:
                    return
            else:
                os.symlink(util.normpath(path), testpath)

            # we've added a builddir file to the testdir,
            # see if there was a symlink in the builddir
            # that pointed to it
            builddirFile = os.path.join(dirName, fileName)
            if builddirFile in self.builddirlinks:
                for path in self.builddirlinks[builddirFile]:
                    linkpath = '%(destdir)s%(thistestdir)s' % self.macros + os.sep + path
                    if not os.path.islink(linkpath):
                        util.mkdirChain(os.path.dirname(linkpath))
                        os.symlink(
                            '%(thistestdir)s/' % self.macros + dirName +
                            os.sep + fileName, linkpath)
Exemplo n.º 13
0
 def _testdirlink(self, path, target):
     fullpath = self._testdirpath(path)
     tpath = self._ddpath(target)
     if os.path.islink(fullpath):
         contents = os.readlink(fullpath)
         if contents[0] == '/':
             return contents == tpath
         else:
             return util.normpath(os.path.join(os.path.dirname(fullpath), contents)) == tpath
     return False
Exemplo n.º 14
0
    def doFile(self, path):
        fullpath = self.macros.destdir + path
        if os.path.islink(fullpath):
            return

        fileName = os.path.basename(path)
        if fileName in self.builddirfiles:
            dirName = self.findRightFile(fullpath, fileName,
                         self.builddirfiles[fileName])
            if dirName is None:
                return

            # if destdir file eq to symlink in builddir
            if os.path.islink(self.macros.builddir + dirName + os.sep
                               + fileName):
                return

            testpath = ('%(destdir)s%(thistestdir)s' + os.sep + dirName
                         + os.sep + fileName) % self.macros
            util.mkdirChain(os.path.dirname(testpath))

            if os.path.islink(testpath):
                buildpath = ''.join((self.macros.builddir, dirName, os.sep, fileName))
                if self.betterLink(self.macros.destdir + path, testpath, buildpath):
                    util.remove(testpath)
                    os.symlink(util.normpath(path), testpath)
                else:
                    return
            else:
                os.symlink(util.normpath(path), testpath)

            # we've added a builddir file to the testdir,
            # see if there was a symlink in the builddir
            # that pointed to it
            builddirFile = os.path.join(dirName, fileName)
            if builddirFile in self.builddirlinks:
                for path in self.builddirlinks[builddirFile]:
                    linkpath = '%(destdir)s%(thistestdir)s' % self.macros + os.sep + path
                    if not os.path.islink(linkpath):
                        util.mkdirChain(os.path.dirname(linkpath))
                        os.symlink('%(thistestdir)s/' % self.macros + dirName
                                    + os.sep + fileName, linkpath)
Exemplo n.º 15
0
 def _getBaseDownloadUrl(self):
     """
     Return the base URL relative to which to download images,
     removing any user/password information, and using http
     """
     # extract the downloadImage base url from the serverUrl configuration
     parts = list(urllib2.urlparse.urlparse(self.rbuilderUrl))
     parts[1] = urllib2.splituser(parts[1])[1]
     # FIXME: is this whole ../ business a workaround for splitbrain rBO?
     parts[2] = parts[2] and parts[2] or "/"
     return "http://%s%s" % (parts[1], util.normpath(parts[2] + "../")[1:])
Exemplo n.º 16
0
 def _getBaseDownloadUrl(self):
     '''
     Return the base URL relative to which to download images,
     removing any user/password information, and using http
     '''
     # extract the downloadImage base url from the serverUrl configuration
     parts = list(urllib2.urlparse.urlparse(self.rbuilderUrl))
     parts[1] = urllib2.splituser(parts[1])[1]
     # FIXME: is this whole ../ business a workaround for splitbrain rBO?
     parts[2] = parts[2] and parts[2] or '/'
     return 'http://%s%s' % (parts[1], util.normpath(parts[2] + '../')[1:])
Exemplo n.º 17
0
 def _gzsymlink(self, dirname, names):
     for name in names:
         path = dirname + os.sep + name
         if os.path.islink(path):
             # change symlinks to .gz -> .gz
             contents = os.readlink(path)
             os.remove(path)
             if not contents.endswith('.gz'):
                 contents = contents + '.gz'
             if not path.endswith('.gz'):
                 path = path + '.gz'
             os.symlink(util.normpath(contents), path)
Exemplo n.º 18
0
 def recordPaths(self, paths):
     if not isinstance(paths, (list, tuple, set)):
         paths = [paths]
     destdir = util.normpath(self.recipe.macros.destdir)
     def _removeDestDir(p):
         p = util.normpath(p)
         if p[:len(destdir)] == destdir:
             return p[len(destdir):]
         else:
             return p
     paths = [_removeDestDir(x % self.recipe.macros) for x in paths]
     self.manifestPaths.update(paths)
Exemplo n.º 19
0
    def addPluggableRequirements(self, path, fullpath, pkgFiles, macros):
        mandatoryReqs, optionalReqs = self._parseEggRequires(path, fullpath)
        filesRequired = []
        for req in itertools.chain(mandatoryReqs, optionalReqs):
            candidatePrefs = [
                '%(destdir)s%(libdir)s/python*/',
                '%(destdir)s%(prefix)s/lib/python*/',
                '%(destdir)s%(libdir)s/python*/site-packages/',
                '%(destdir)s%(prefix)s/lib/python*/site-packages/',
                '%(libdir)s/python*/',
                '%(prefix)s/lib/python*/',
                '%(libdir)s/python*/site-packages/',
                '%(prefix)s/lib/python*/site-packages/',
            ]
            candidateFileNames = [(x + req + '*.egg-info/PKG-INFO') % macros \
                    for x in candidatePrefs]
            candidateFiles = [fixedglob.glob(x) for x in candidateFileNames]
            candidateFiles = [x[0] for x in candidateFiles if x]
            if candidateFiles:
                filesRequired.append(candidateFiles[0])
            else:
                if req in mandatoryReqs:
                    self.warn('Python egg-info for %s was not found', req)

        for fileRequired in filesRequired:
            troveName = None
            if fileRequired.startswith(macros.destdir):
                # find requirement in packaging
                fileRequired = util.normpath(os.path.realpath(fileRequired))
                fileRequired = fileRequired[len(util.normpath(macros.destdir)
                                                ):]
                autopkg = self.recipe.autopkg
                troveName = autopkg.findComponent(fileRequired).getName()
            else:
                troveName = self._enforceProvidedPath(fileRequired,
                                                      fileType='egg-info',
                                                      unmanagedError=True)
            if troveName:
                self._addRequirement(path, troveName, [], pkgFiles,
                                     deps.TroveDependencies)
Exemplo n.º 20
0
    def doFile(self, path):
        if hasattr(self.recipe, '_getCapsulePathsForFile'):
            if self.recipe._getCapsulePathsForFile(path):
                return

        fullpath = self.macros['destdir'] + path
        if os.path.islink(fullpath):
            contents = os.readlink(fullpath)
            if contents.startswith('/'):
                pathlist = util.normpath(path).split('/')
                contentslist = util.normpath(contents).split('/')
                if pathlist == contentslist:
                    raise policy.PolicyError('Symlink points to itself:'
                                             ' %s -> %s' % (path, contents))
                while contentslist and pathlist[0] == contentslist[0]:
                    pathlist = pathlist[1:]
                    contentslist = contentslist[1:]
                os.remove(fullpath)
                dots = "../"
                dots *= len(pathlist) - 1
                normpath = util.normpath(dots + '/'.join(contentslist))
                os.symlink(normpath, fullpath)
Exemplo n.º 21
0
    def addPluggableRequirements(self, path, fullpath, pkgFiles, macros):
        mandatoryReqs, optionalReqs = self._parseEggRequires(path, fullpath)
        filesRequired = []
        for req in itertools.chain(mandatoryReqs, optionalReqs):
            candidatePrefs = [
                '%(destdir)s%(libdir)s/python*/',
                '%(destdir)s%(prefix)s/lib/python*/',
                '%(destdir)s%(libdir)s/python*/site-packages/',
                '%(destdir)s%(prefix)s/lib/python*/site-packages/',
                '%(libdir)s/python*/',
                '%(prefix)s/lib/python*/',
                '%(libdir)s/python*/site-packages/',
                '%(prefix)s/lib/python*/site-packages/',
                    ]
            candidateFileNames = [(x + req + '*.egg-info/PKG-INFO') % macros \
                    for x in candidatePrefs]
            candidateFiles = [fixedglob.glob(x) for x in candidateFileNames]
            candidateFiles = [x[0] for x in candidateFiles if x]
            if candidateFiles:
                filesRequired.append(candidateFiles[0])
            else:
                if req in mandatoryReqs:
                    self.warn('Python egg-info for %s was not found', req)

        for fileRequired in filesRequired:
            troveName = None
            if fileRequired.startswith(macros.destdir):
                # find requirement in packaging
                fileRequired = util.normpath(os.path.realpath(fileRequired))
                fileRequired = fileRequired[len(util.normpath(macros.destdir)):]
                autopkg = self.recipe.autopkg
                troveName = autopkg.findComponent(fileRequired).getName()
            else:
                troveName = self._enforceProvidedPath(fileRequired,
                                                      fileType='egg-info',
                                                      unmanagedError=True)
            if troveName:
                self._addRequirement(path, troveName, [], pkgFiles,
                                     deps.TroveDependencies)
Exemplo n.º 22
0
def sortMountPoints(mounts):
    """Sorts mount points by specificity by counting os.path.sep instances."""
    mounts = [util.normpath(x) for x in mounts]

    # sort mounts by specificity, more specific mountpoints first.
    mounts = sorted(mounts, key = lambda x: x.count(os.path.sep), reverse = True)

    # / is special, put it at the end
    if os.path.sep in mounts:
        mounts.remove(os.path.sep)
        mounts.append(os.path.sep)

    return mounts
Exemplo n.º 23
0
    def doFile(self, path):
        if hasattr(self.recipe, '_getCapsulePathsForFile'):
            if self.recipe._getCapsulePathsForFile(path):
                return

        fullpath = self.macros['destdir']+path
        if os.path.islink(fullpath):
            contents = os.readlink(fullpath)
            if contents.startswith('/'):
                pathlist = util.normpath(path).split('/')
                contentslist = util.normpath(contents).split('/')
                if pathlist == contentslist:
                    raise policy.PolicyError('Symlink points to itself:'
                                             ' %s -> %s' % (path, contents))
                while contentslist and pathlist[0] == contentslist[0]:
                    pathlist = pathlist[1:]
                    contentslist = contentslist[1:]
                os.remove(fullpath)
                dots = "../"
                dots *= len(pathlist) - 1
                normpath = util.normpath(dots + '/'.join(contentslist))
                os.symlink(normpath, fullpath)
Exemplo n.º 24
0
    def recordPaths(self, paths):
        if not isinstance(paths, (list, tuple, set)):
            paths = [paths]
        destdir = util.normpath(self.recipe.macros.destdir)

        def _removeDestDir(p):
            p = util.normpath(p)
            if p[:len(destdir)] == destdir:
                return p[len(destdir):]
            else:
                return p

        paths = [_removeDestDir(x % self.recipe.macros) for x in paths]
        self.manifestPaths.update(paths)
Exemplo n.º 25
0
 def postInit(self):
     self.runnable = True
     self.warnedSoNames = set()
     self.logLines = []
     # subscribe to necessary build log entries
     if hasattr(self.recipe, 'subscribeLogs'):
         macros = {'cc': re.escape(self.recipe.macros.cc),
                   'cxx': re.escape(self.recipe.macros.cxx)}
         regexp = self.regexp % macros
         self.recipe.subscribeLogs(regexp)
         self.r = re.compile(regexp)
         macros = self.recipe.macros
         cfg = self.recipe.cfg
         self.libDirs = {'%s%s' %(cfg.root, macros.libdir): macros.libdir,
                         util.normpath('%s/%s'%(cfg.root, macros.lib)): '/%s' %macros.lib}
         self._initComponentExceptions()
     else:
         # disable this policy
         self.runnable = False
Exemplo n.º 26
0
    def recordMove(self, src, dest):
        destdir = util.normpath(self.macros.destdir)

        def _removeDestDir(p):
            p = util.normpath(p)
            if p[:len(destdir)] == destdir:
                return p[len(destdir):]
            else:
                return p

        if os.path.isdir(src):
            # assume move is about to happen
            baseDir = src
            postRename = False
        elif os.path.isdir(dest):
            # assume move just happened
            baseDir = dest
            postRename = True
        else:
            # don't walk directories
            baseDir = None
        src = _removeDestDir(src)
        dest = _removeDestDir(dest)
        self._pathTranslations.append((src, dest))
        if baseDir:
            for base, dirs, files in os.walk(baseDir):
                for path in dirs + files:
                    if not postRename:
                        fSrc = os.path.join(base, path)
                        fSrc = fSrc.replace(self.macros.destdir, '')
                        fDest = fSrc.replace(src, dest)
                    else:
                        fDest = os.path.join(base, path)
                        fDest = fDest.replace(self.macros.destdir, '')
                        fSrc = fDest.replace(dest, src)
                    self._pathTranslations.append((fSrc, fDest))
Exemplo n.º 27
0
 def _removeDestDir(p):
     p = util.normpath(p)
     if p[:len(destdir)] == destdir:
         return p[len(destdir):]
     else:
         return p
Exemplo n.º 28
0
def getTrovesToDisplay(db, troveSpecs, pathList=[], whatProvidesList=[],
                       exactFlavors=False):
    """ Finds the given trove and path specifiers, and returns matching
        (n,v,f) tuples.
        @param db: database to search
        @type db: local.database.Database
        @param troveSpecs: troves to search for
        @type troveSpecs: list of troveSpecs (n[=v][[f]])
        @param pathList: paths which should be linked to some trove in this
                         database.
        @type pathList: list of strings
        @param whatProvidesList: deps to search for providers of
        @type whatProvidesList: list of strings

        @raises TroveSpecError: Raised if one of the troveSpecs is of an
                                invalid format

        @note: This function calls database routines which could raise any
               errors defined in L{dbstore.sqlerrors}

        @rtype: troveTupleList (list of (name, version, flavor) tuples),
                and a boolean that stats whether the troves returned should
                be considered primary (and therefore not compressed ever).
    """

    primary = True

    if troveSpecs:
        troveSpecs = [ cmdline.parseTroveSpec(x, allowEmptyName=False) \
                                                        for x in troveSpecs ]
    else:
        troveSpecs = []

    normPathList = [ util.realpath(os.path.abspath(util.normpath(x)))
                     for x in pathList ]

    troveTups = []
    for path, origPath in itertools.izip(normPathList, pathList):
        if origPath.endswith('/'):
            allPaths = [ path + '/' + x for x in os.listdir(db.root + path) ]
        else:
            allPaths = [ path ]

        for thisPath in allPaths:
            for trove in db.iterTrovesByPath(thisPath):
                troveTups.append((trove.getName(), trove.getVersion(),
                                  trove.getFlavor()))

    if whatProvidesList:
        results = db.getTrovesWithProvides(whatProvidesList)
        troveTups.extend(itertools.chain(*results.itervalues()))

    if not (troveSpecs or pathList or whatProvidesList):
        troveTups = sorted(db.iterAllTroves())
        primary = False
    else:
        results = db.findTroves(None, troveSpecs, exactFlavors=exactFlavors)

        for troveSpec in troveSpecs:
            troveTups.extend(results.get(troveSpec, []))

    return troveTups, primary
Exemplo n.º 29
0
    def doProcess(self, recipe):
        """
        Invocation instance
        @param recipe: holds the recipe object, which is used for
        the macro set and package objects.
        @return: None
        @rtype: None
        """
        self.recipe = recipe
        self.macros = recipe.macros

        if not self._isSupportedTarget():
            return

        if self.rootdir:
            self.rootdir = util.normpath(self.rootdir % self.macros)

        if (hasattr(recipe, '_isDerived')
            and recipe._isDerived == True
            and self.processUnmodified is None):
            # This policy does not handle derived packages
            return

        if hasattr(self.__class__, 'preProcess'):
            self.preProcess()

        # is runtime check implemented?
        if hasattr(self.__class__, 'test'):
            if not self.test():
                return

        # change self.use to be a simple flag
        self.use = action.checkUse(self.use)

        # compile the exceptions
        self.exceptionFilters = []
        self.compileFilters(self.invariantexceptions, self.exceptionFilters)
        if self.exceptions:
            if not isinstance(self.exceptions, (tuple, list, set)):
                # turn a plain string into a sequence
                self.exceptions = (self.exceptions,)
            self.compileFilters(self.exceptions, self.exceptionFilters, self.unusedFilters['exceptions'])

        # compile the inclusions
        self.inclusionFilters = []
        if self.invariantinclusions is None:
            self.compileFilters([], self.inclusionFilters)
        else:
            self.compileFilters(self.invariantinclusions, self.inclusionFilters)
        if not self.inclusions:
            # an empty list, as opposed to None, means nothing is included
            if isinstance(self.inclusions, (tuple, list, set)):
                return
        else:
            if not isinstance(self.inclusions, (tuple, list, set)):
                # turn a plain string into a sequence
                self.inclusions = (self.inclusions,)
            self.compileFilters(self.inclusions, self.inclusionFilters, self.unusedFilters['inclusions'])

        # dispatch if/as appropriate
        if self.use:
            self.do()

        if hasattr(self.__class__, 'postProcess'):
            self.postProcess()
Exemplo n.º 30
0
def _normpath(path):
    return util.normpath(path).lstrip('/')
Exemplo n.º 31
0
    def do(self):
        if hasattr(self.recipe, '_getCapsulePathsForFile'):
            if self.recipe.getType() == recipe.RECIPE_TYPE_CAPSULE:
                return
        macros = self.macros
        subtrees = self.invariantsubtrees
        if self.subtrees:
            subtrees.extend(self.subtrees)
        for path in subtrees:
            path = util.normpath(path % macros)
            fullpath = '/'.join((self.macros.destdir, path))
            if not os.path.exists(fullpath):
                continue
            # this state can only be reached if SharedLibrary is called with
            # bad arguments... see CNP-45
            mode = os.stat(fullpath)[stat.ST_MODE]
            if not stat.S_ISDIR(mode):
                self.error(
                    'The subtrees= argument takes directories only;'
                    ' %s is not a directory', path)
                continue
            oldfiles = set(os.listdir(fullpath))
            bootStrapLdConfig = True
            ldConfigPath = '%(destdir)s%(essentialsbindir)s/ldconfig' % macros
            if (not os.path.exists(ldConfigPath)
                ) or self.recipe.isCrossCompiling():
                bootStrapLdConfig = False
                ldConfigPath = '%(essentialsbindir)s/ldconfig' % macros
            if not os.path.exists(ldConfigPath):
                self.warn('ldconfig not found')
                continue
            util.execute('%s -n %s' % (ldConfigPath, fullpath))

            if not bootStrapLdConfig:
                db = database.Database(self.recipe.cfg.root,
                                       self.recipe.cfg.dbPath)
                ldConfigTroveName = [
                    x.getName() for x in db.iterTrovesByPath(ldConfigPath)
                ]
                if ldConfigTroveName:
                    ldConfigTroveName = ldConfigTroveName[0]
                else:
                    ldConfigTroveName = 'glibc:runtime'

                try:
                    if ldConfigTroveName in self.recipe._getTransitiveBuildRequiresNames(
                    ):
                        self.recipe.reportExcessBuildRequires(
                            ldConfigTroveName)
                    else:
                        self.recipe.reportMissingBuildRequires(
                            ldConfigTroveName)
                except AttributeError:
                    # older Conary that does not have
                    # reportExcessBuildRequires or even the older
                    # reportMissingBuildRequires or
                    # _getTransitiveBuildRequiresNames
                    pass

            newfiles = set(os.listdir(fullpath))
            addedfiles = newfiles - oldfiles
            removedfiles = oldfiles - newfiles
            if addedfiles:
                self.info('ldconfig added the following new files in %s: %s',
                          path, ', '.join(sorted(list(addedfiles))))
            if removedfiles:
                self.warn('ldconfig removed files in %s: %s', path,
                          ', '.join(sorted(list(removedfiles))))
Exemplo n.º 32
0
    def testLookAsideTest1(self):
        raise testhelp.SkipTestException("tests shouldn't talk to the internet")
        self.resetCache()
        self.resetWork()
        self.resetRepository()
        repos = self.openRepository()
        cfg = self.cfg
        repCache = lookaside.RepositoryCache(repos, cfg=cfg)

        origDir = os.getcwd()
        os.chdir(self.workDir)
        self.newpkg('test')
        os.chdir('test')
        testdir = '/'.join((self.workDir, 'test'))

        log.setVerbosity(log.INFO)
        self.logFilter.add()
        # first, look for a file that does not exist
        assert(lookaside.findAll(self.cfg, repCache,
            'http://example.conary.com/foo', 'test', (testdir,),
            allowNone=True) is None)
        # make sure that we got a negative cache entry
        assert(os.stat('/'.join((self.cacheDir, 'NEGATIVE/test/example.conary.com/foo'))))

        # now make sure that this works for ftp as well (urllib workaround)
        #    XXX WORKAROUND until FTP works in eng lab
        #assert(lookaside.findAll(self.cfg, repCache,
        #    'ftp://download.rpath.com/blah', 'test', (testdir,),
        #     allowNone=True) is None)
        # make sure that we got a negative cache entry
        #assert(os.stat('/'.join((self.cacheDir,
        #                         'NEGATIVE/test/download.rpath.com/blah'))))

        # now we put a file in place
        f = file(os.sep.join((testdir, 'bar')), 'w')
        f.write('this is a test')
        f.close()
        c = lookaside.findAll(self.cfg, repCache, 'bar', 'test', (testdir,))
        # it does not need to cache it; it is known to exist
        assert(c == os.sep.join((testdir, 'bar')))
        
        # Test httpHeaders:
        c = util.normpath(lookaside.findAll(self.cfg, repCache,
            'http://www.google.com/preferences', 'test', (testdir,),
            httpHeaders={'Accept-Language': 'es-es'}))
        assert(c == '/'.join((self.cacheDir, 'test/www.google.com/preferences')))
        #open the page and check to see if it's in spanish
        f = open(c)
        contents = f.read()
        f.close()
        assert 'Preferencias globales' in contents

        # we need a web page to actually test the cache in operation
        # we do it a second time to make sure that the cache works
        for i in (0, 1):
            c = util.normpath(lookaside.findAll(self.cfg, repCache,
                'http://wiki.rpath.com/wiki/Main_Page', 'test', (testdir,)))
            assert(c == '/'.join((self.cacheDir, 'test/wiki.rpath.com/wiki/Main_Page')))
        self.logFilter.remove()
        self.logFilter.compare(
            ('+ Trying http://example.conary.com/foo...',
        # XXX WORKAROUND until FTP works in eng lab
             #'+ Trying ftp://download.rpath.com/blah...',
             #'+ Downloading ftp://download.rpath.com/blah...',
             '+ Trying http://www.google.com/preferences...',
             '+ Downloading http://www.google.com/preferences...',
             '+ Trying http://wiki.rpath.com/wiki/Main_Page...',
             '+ Downloading http://wiki.rpath.com/wiki/Main_Page...'))

        recipestr = """
class TestLookaside(PackageRecipe):
    name = 'test'
    version = '1'
    clearBuildReqs()

    def setup(r):
        r.addSource('bar', dest='/')
"""
        self.writeFile('test.recipe', recipestr)
        self.addfile('test.recipe')
        self.addfile('bar', binary = True)
        self.commit()
        os.chdir(origDir)

        # ensure that a localOnly=True lookup in the repository works;
        # for this, we need a prepped recipeObj for its RepositoryCache
        # object
        recipeObj = self.getRecipeObjFromRepos('test', repos)
        self.logFilter.add()
        c = lookaside.findAll(self.cfg, recipeObj.laReposCache,
                              'bar', 'test', (), localOnly=True)
        self.logFilter.remove()
        self.logFilter.compare(
            '+ found bar in repository',
        )
        assert(c == os.sep.join((self.cacheDir, 'test', 'bar')))
Exemplo n.º 33
0
    def install(self, flags, troveCs):
        ACTION_RESTORE = 1
        ACTION_SKIP = 2
        ACTION_CONFLICT = 3

        rc = SingleCapsuleOperation.install(self, flags, troveCs)
        if rc is None:
            # parent class thinks we should just ignore this troveCs; I'm
            # not going to argue with it (it's probably because the capsule
            # hasn't changed
            return None

        (oldTrv, trv) = rc
        trvInfo = troveCs.getNewNameVersionFlavor()
        oldTrvInfo = troveCs.getOldNameVersionFlavor()
        hasCapsule = troveCs.hasCapsule()

        # Updates the fsJob metadata for installing the current trove.
        # It assumes files are replaced on install, and complains if something
        # is in the way unless the appropriate flags are set. This is a very
        # much simplified version of FilesystemJob._singleTrove() which maps
        # out a complete install strategy for native packages. Note that
        # we walk all of the files in this trove, not just the new files
        # or the changed files, because RPM installs all of the files.
        toRestore = []

        changedByPathId = dict((x[0], x) for x in troveCs.getChangedFileList())

        # things which aren't change, new, or removed are unchanged
        unchangedByPathId = (set(x[0] for x in trv.iterFileList()) -
                             set(changedByPathId.iterkeys()) -
                             set(x[0] for x in troveCs.getNewFileList()) -
                             set(troveCs.getOldFileList()))

        l = []
        for oldFileInfo in troveCs.getChangedFileList():
            oldFileId, oldVersion = oldTrv.getFile(oldFileInfo[0])[1:3]
            l.append((oldFileInfo[0], oldFileId, oldVersion))

        for unchangedPathId in unchangedByPathId:
            unchangedFileId, unchangedFileVersion = \
                                    trv.getFile(unchangedPathId)[1:3]
            l.append((unchangedPathId, unchangedFileId, unchangedFileVersion))

        fileObjs = self.db.getFileVersions(l)
        fileObjsByPathId = dict(
                [ (x[0], y) for x, y in
                    itertools.izip(l, fileObjs) ] )

        for fileInfo in trv.iterFileList():
            pathId, path, fileId, version = fileInfo

            if os.path.dirname(path) in self.netSharedPath:
                # we do nothing. really. nothing.
                #
                # we don't back it up. we don't mark it as removed in
                # our database. we don't look for conflicts. nothing.
                continue

            if pathId in changedByPathId:
                oldFileId = oldTrv.getFile(pathId)[1]
                fileChange = self.changeSet.getFileChange(oldFileId, fileId)
                if (oldFileId == fileId):
                    # only the version number changed; we don't need
                    # to merge anything here
                    fileObj = fileObjsByPathId[pathId]
                elif fileChange[0] == '\x01':
                    fileObj = fileObjsByPathId[pathId]
                    fileObj.twm(fileChange, fileObj)
                else:
                    fileObj = files.ThawFile(fileChange, pathId)
            elif pathId in unchangedByPathId:
                fileObj = fileObjsByPathId[pathId]
            else:
                # if it's not changed and it's not unchanged, it must be new
                fileStream = self.changeSet.getFileChange(None, fileId)
                fileObj = files.ThawFile(fileStream, pathId)

            absolutePath = util.joinPaths(self.root, path)

            if (fileObj.flags.isCapsuleAddition()):
                # this was added to the package outside of the RPM; we don't
                # have any responsibility for it
                continue
            elif (trove.conaryContents(hasCapsule, pathId, fileObj)
                  and fileObj.lsTag != 'd'):
                # this content isn't part of the capsule; remember to put
                # it back when RPM is done
                self.preservePath(path)
                continue

            s = util.lstat(absolutePath)
            if not s:
                # there is nothing in the way, so there is nothing which
                # concerns us here. Track the file for later.
                toRestore.append((fileInfo, fileObj))
                continue

            action = ACTION_CONFLICT

            existingOwners = list(
                self.db.iterFindPathReferences(path, justPresent = True,
                                               withStream = True))

            if existingOwners:
                # Don't complain about files owned by the previous version
                # of this trove.
                l = [ x for x in existingOwners if x[0:3] == oldTrvInfo ]
                if l:
                    existingOwners.remove(l[0])

                if not existingOwners:
                    action = ACTION_RESTORE
            elif stat.S_ISDIR(s.st_mode) and fileObj.lsTag == 'd':
                # Don't let existing directories stop us from taking over
                # ownership of the directory
                action = ACTION_RESTORE
            elif fileObj.flags.isInitialContents():
                # Initial contents files may be restored on top of things
                # already in the filesystem. They're ghosts or config files
                # and RPM will get the contents right either way, and we
                # should remove them either way.
                action = ACTION_RESTORE

            if action == ACTION_CONFLICT and existingOwners:
                if fileId in [ x[4] for x in existingOwners ]:
                    # The files share metadata same. Whatever it looks like on
                    # disk, RPM is going to blow it away with the new one.
                    for info in existingOwners:
                        self.fsJob.sharedFile(info[0], info[1], info[2],
                                              info[3])
                    action = ACTION_RESTORE
                elif path.startswith('/usr/share/doc/'):
                    # Mirror badness Red Hat patches into RPM for rhel4
                    # and rhel5
                    action = ACTION_RESTORE
                else:
                    existingFiles = [ files.ThawFile(x[5], pathId) for x
                                        in existingOwners ]

                    compatibility = [ 1 for x in existingFiles
                                        if fileObj.compatibleWith(x) ]


                    if 1 in compatibility:
                        # files can be shared even though the fileId's
                        # are different
                        for info in existingOwners:
                            self.fsJob.sharedFile(info[0], info[1], info[2],
                                                  info[3])
                        action = ACTION_RESTORE
                    elif 1 in [ files.rpmFileColorCmp(x, fileObj)
                              for x in existingFiles ]:
                        # rpm file colors and the default rpm setting for
                        # file color policy make elf64 files silently replace
                        # elf32 files. follow that behavior here.
                        #
                        # no, i'm not making this up
                        #
                        # yes, really
                        action = ACTION_SKIP
                    elif (self._checkReplaceManagedFiles(flags, path) or
                          1 in [ files.rpmFileColorCmp(fileObj, x)
                                 for x in existingFiles ]):
                        # The files are different. Bail unless we're supposed
                        # to replace managed files.
                        existingFile = files.FileFromFilesystem(absolutePath,
                                                                pathId)
                        for info in existingOwners:
                            self.fsJob.userRemoval(
                                fileObj = existingFile,
                                content =
                                    filecontents.FromFilesystem(absolutePath),
                                *info[0:4])
                        action = ACTION_RESTORE
                    else:
                        # it's not up to us to decide if this is a true
                        # conflict; the database layer will do that for
                        # us (see checkPathConflicts)
                        action = ACTION_RESTORE
            elif flags.replaceUnmanagedFiles:
                # we don't own it, but it's on disk. RPM will just write over
                # it and we have the flag saying we're good with that
                action = ACTION_RESTORE

            if action == ACTION_RESTORE:
                # We may proceed, and RPM will replace this file for us. We
                # need to track that it's being restored to avoid conflicts
                # with other restorations though.
                toRestore.append((fileInfo, fileObj))
            elif action == ACTION_CONFLICT:
                # The file exists already, we can't share it, and we're not
                # allowed to overwrite it.
                self._error(errors.FileInWayError(util.normpath(path),
                                                  troveCs.getName(),
                                                  troveCs.getNewVersion(),
                                                  troveCs.getNewFlavor()))
            else:
                assert(action == ACTION_SKIP)
                self.preservePath(path)
                self.fsJob.userRemoval(trv.getName(), trv.getVersion(),
                                       trv.getFlavor(), pathId)

        # toRestore is the list of what is going to be restored. We need to get
        # the fileObjects which will be created so we can track them in the
        # filesystem job. This lets the filesystem job look for resolveable
        # conflicts within this update. We handle newly created files first
        # and files which have changed (so we have to look up the diff)
        # a bit later.
        for fileInfo, fileObj in toRestore:
            fullPath = util.joinPaths(self.root, path)
            self.fsJob._restore(fileObj, fullPath, trvInfo,
                                "restoring %s from RPM",
                                restoreFile = False,
                                fileId = fileId)
Exemplo n.º 34
0
    def addPluggableRequirements(self, path, fullpath, pkgFiles, macros):
        if hasattr(self.recipe, '_getCapsulePathsForFile'):
            if self.recipe._getCapsulePathsForFile(path):
                # since capsules do not convert to relative symlinks,
                # we cannot depend on getting the realpath.  Unless
                # we resolve that, assume that capsule-provided
                # dependencies will be sufficient for pkgconfig files.
                return

        # parse pkgconfig file
        variables = {}
        requirements = set()
        libDirs = []
        libraries = set()
        variableLineRe = re.compile('^[a-zA-Z0-9]+=')
        filesRequired = []

        pcContents = [x.strip() for x in file(fullpath).readlines()]
        for pcLine in pcContents:
            # interpolate variables: assume variables are interpreted
            # line-by-line while processing
            pcLineIter = pcLine
            while True:
                for var in variables:
                    pcLineIter = pcLineIter.replace(var, variables[var])
                if pcLine == pcLineIter:
                    break
                pcLine = pcLineIter
            pcLine = pcLineIter

            if variableLineRe.match(pcLine):
                key, val = pcLine.split('=', 1)
                variables['${%s}' %key] = val
            else:
                if (pcLine.startswith('Requires') or
                    pcLine.startswith('Lib')) and ':' in pcLine:
                    keyWord, args = pcLine.split(':', 1)
                    # split on ',' and ' '
                    argList = itertools.chain(*[x.split(',')
                                                for x in args.split()])
                    argList = [x for x in argList if x]
                    if keyWord.startswith('Requires'):
                        versionNext = False
                        for req in argList:
                            if [x for x in '<=>' if x in req]:
                                versionNext = True
                                continue
                            if versionNext:
                                versionNext = False
                                continue
                            requirements.add(req)
                    elif keyWord.startswith('Lib'):
                        for lib in argList:
                            if lib.startswith('-L'):
                                libDirs.append(lib[2:])
                            elif lib.startswith('-l'):
                                libraries.add(lib[2:])
                            else:
                                pass

        # find referenced pkgconfig files and add requirements
        for req in requirements:
            candidateFileNames = [
                '%(destdir)s%(libdir)s/pkgconfig/'+req+'.pc',
                '%(destdir)s%(datadir)s/pkgconfig/'+req+'.pc',
                '%(libdir)s/pkgconfig/'+req+'.pc',
                '%(datadir)s/pkgconfig/'+req+'.pc',
            ]
            candidateFileNames = [ x % macros for x in candidateFileNames ]
            candidateFiles = [ util.exists(x) for x in candidateFileNames ]
            if True in candidateFiles:
                filesRequired.append(
                    (candidateFileNames[candidateFiles.index(True)], 'pkg-config'))
            else:
                self.warn('pkg-config file %s.pc not found', req)
                continue

        # find referenced library files and add requirements
        libraryPaths = sorted(list(self.systemLibPaths))
        for libDir in libDirs:
            if libDir not in libraryPaths:
                libraryPaths.append(libDir)
        for library in libraries:
            found = False
            for libDir in libraryPaths:
                candidateFileNames = [
                    macros.destdir+libDir+'/lib'+library+'.so',
                    macros.destdir+libDir+'/lib'+library+'.a',
                    libDir+'/lib'+library+'.so',
                    libDir+'/lib'+library+'.a',
                ]
                candidateFiles = [ util.exists(x) for x in candidateFileNames ]
                if True in candidateFiles:
                    filesRequired.append(
                        (candidateFileNames[candidateFiles.index(True)], 'library'))
                    found = True
                    break

            if not found:
                self.warn('library file lib%s not found', library)
                continue


        for fileRequired, fileType in filesRequired:
            if fileRequired.startswith(macros.destdir):
                # find requirement in packaging
                fileRequired = util.normpath(os.path.realpath(fileRequired))
                fileRequired = fileRequired[len(util.normpath(os.path.realpath(macros.destdir))):]
                autopkg = self.recipe.autopkg
                troveName = autopkg.componentMap[fileRequired].name
                package, component = troveName.split(':', 1)
                if component in ('devellib', 'lib'):
                    for preferredComponent in ('devel', 'devellib'):
                        develTroveName = ':'.join((package, preferredComponent))
                        if develTroveName in autopkg.components and autopkg.components[develTroveName]:
                            # found a non-empty :devel compoment
                            troveName = develTroveName
                            break
                self._addRequirement(path, troveName, [], pkgFiles,
                                     deps.TroveDependencies)
            else:
                troveName = self._enforceProvidedPath(fileRequired,
                                                      fileType=fileType,
                                                      unmanagedError=True)
                if troveName:
                    self._addRequirement(path, troveName, [], pkgFiles,
                                         deps.TroveDependencies)
Exemplo n.º 35
0
 def _removeDestDir(p):
     p = util.normpath(p)
     if p[:len(destdir)] == destdir:
         return p[len(destdir):]
     else:
         return p
Exemplo n.º 36
0
    def testLookAsideTest1(self):
        raise testhelp.SkipTestException(
            "tests shouldn't talk to the internet")
        self.resetCache()
        self.resetWork()
        self.resetRepository()
        repos = self.openRepository()
        cfg = self.cfg
        repCache = lookaside.RepositoryCache(repos, cfg=cfg)

        origDir = os.getcwd()
        os.chdir(self.workDir)
        self.newpkg('test')
        os.chdir('test')
        testdir = '/'.join((self.workDir, 'test'))

        log.setVerbosity(log.INFO)
        self.logFilter.add()
        # first, look for a file that does not exist
        assert (lookaside.findAll(self.cfg,
                                  repCache,
                                  'http://example.conary.com/foo',
                                  'test', (testdir, ),
                                  allowNone=True) is None)
        # make sure that we got a negative cache entry
        assert (os.stat('/'.join(
            (self.cacheDir, 'NEGATIVE/test/example.conary.com/foo'))))

        # now make sure that this works for ftp as well (urllib workaround)
        #    XXX WORKAROUND until FTP works in eng lab
        #assert(lookaside.findAll(self.cfg, repCache,
        #    'ftp://download.rpath.com/blah', 'test', (testdir,),
        #     allowNone=True) is None)
        # make sure that we got a negative cache entry
        #assert(os.stat('/'.join((self.cacheDir,
        #                         'NEGATIVE/test/download.rpath.com/blah'))))

        # now we put a file in place
        f = file(os.sep.join((testdir, 'bar')), 'w')
        f.write('this is a test')
        f.close()
        c = lookaside.findAll(self.cfg, repCache, 'bar', 'test', (testdir, ))
        # it does not need to cache it; it is known to exist
        assert (c == os.sep.join((testdir, 'bar')))

        # Test httpHeaders:
        c = util.normpath(
            lookaside.findAll(self.cfg,
                              repCache,
                              'http://www.google.com/preferences',
                              'test', (testdir, ),
                              httpHeaders={'Accept-Language': 'es-es'}))
        assert (c == '/'.join(
            (self.cacheDir, 'test/www.google.com/preferences')))
        #open the page and check to see if it's in spanish
        f = open(c)
        contents = f.read()
        f.close()
        assert 'Preferencias globales' in contents

        # we need a web page to actually test the cache in operation
        # we do it a second time to make sure that the cache works
        for i in (0, 1):
            c = util.normpath(
                lookaside.findAll(self.cfg, repCache,
                                  'http://wiki.rpath.com/wiki/Main_Page',
                                  'test', (testdir, )))
            assert (c == '/'.join(
                (self.cacheDir, 'test/wiki.rpath.com/wiki/Main_Page')))
        self.logFilter.remove()
        self.logFilter.compare((
            '+ Trying http://example.conary.com/foo...',
            # XXX WORKAROUND until FTP works in eng lab
            #'+ Trying ftp://download.rpath.com/blah...',
            #'+ Downloading ftp://download.rpath.com/blah...',
            '+ Trying http://www.google.com/preferences...',
            '+ Downloading http://www.google.com/preferences...',
            '+ Trying http://wiki.rpath.com/wiki/Main_Page...',
            '+ Downloading http://wiki.rpath.com/wiki/Main_Page...'))

        recipestr = """
class TestLookaside(PackageRecipe):
    name = 'test'
    version = '1'
    clearBuildReqs()

    def setup(r):
        r.addSource('bar', dest='/')
"""
        self.writeFile('test.recipe', recipestr)
        self.addfile('test.recipe')
        self.addfile('bar', binary=True)
        self.commit()
        os.chdir(origDir)

        # ensure that a localOnly=True lookup in the repository works;
        # for this, we need a prepped recipeObj for its RepositoryCache
        # object
        recipeObj = self.getRecipeObjFromRepos('test', repos)
        self.logFilter.add()
        c = lookaside.findAll(self.cfg,
                              recipeObj.laReposCache,
                              'bar',
                              'test', (),
                              localOnly=True)
        self.logFilter.remove()
        self.logFilter.compare('+ found bar in repository', )
        assert (c == os.sep.join((self.cacheDir, 'test', 'bar')))
Exemplo n.º 37
0
def _normpath(path):
    return util.normpath(path).lstrip('/')
Exemplo n.º 38
0
    def doProcess(self, recipe):
        """
        Invocation instance
        @param recipe: holds the recipe object, which is used for
        the macro set and package objects.
        @return: None
        @rtype: None
        """
        self.recipe = recipe
        self.macros = recipe.macros

        if not self._isSupportedTarget():
            return

        if self.rootdir:
            self.rootdir = util.normpath(self.rootdir % self.macros)

        if (hasattr(recipe, '_isDerived') and recipe._isDerived == True
                and self.processUnmodified is None):
            # This policy does not handle derived packages
            return

        if hasattr(self.__class__, 'preProcess'):
            self.preProcess()

        # is runtime check implemented?
        if hasattr(self.__class__, 'test'):
            if not self.test():
                return

        # change self.use to be a simple flag
        self.use = action.checkUse(self.use)

        # compile the exceptions
        self.exceptionFilters = []
        self.compileFilters(self.invariantexceptions, self.exceptionFilters)
        if self.exceptions:
            if not isinstance(self.exceptions, (tuple, list, set)):
                # turn a plain string into a sequence
                self.exceptions = (self.exceptions, )
            self.compileFilters(self.exceptions, self.exceptionFilters,
                                self.unusedFilters['exceptions'])

        # compile the inclusions
        self.inclusionFilters = []
        if self.invariantinclusions is None:
            self.compileFilters([], self.inclusionFilters)
        else:
            self.compileFilters(self.invariantinclusions,
                                self.inclusionFilters)
        if not self.inclusions:
            # an empty list, as opposed to None, means nothing is included
            if isinstance(self.inclusions, (tuple, list, set)):
                return
        else:
            if not isinstance(self.inclusions, (tuple, list, set)):
                # turn a plain string into a sequence
                self.inclusions = (self.inclusions, )
            self.compileFilters(self.inclusions, self.inclusionFilters,
                                self.unusedFilters['inclusions'])

        # dispatch if/as appropriate
        if self.use:
            self.do()

        if hasattr(self.__class__, 'postProcess'):
            self.postProcess()
Exemplo n.º 39
0
    def do(self):
        # For the purposes of this policy, the transitive buildRequires
        # includes suggestions already made for handling shared libraries,
        # since this policy is explicitly a fallback for the unusual
        # case of static linking outside of the package being built.
        transitiveBuildRequires = self.transitiveBuildRequires.union(self.warnedSoNames)
        cfg = self.recipe.cfg
        db = database.Database(cfg.root, cfg.dbPath)

        foundLibNames = set()
        allPossibleProviders = set()
        missingBuildRequires = set()
        self.buildDirLibNames = None
        destdir = self.recipe.macros.destdir
        builddir = self.recipe.macros.builddir
        tooManyChoices = {}
        noTroveFound = {}
        noLibraryFound = set()

        components = self.recipe.autopkg.components
        pathMap = self.recipe.autopkg.pathMap
        reqDepSet = deps.DependencySet()
        sharedLibraryRequires = set()
        for pkg in components.values():
            reqDepSet.union(pkg.requires)
        for dep in reqDepSet.iterDepsByClass(deps.SonameDependencies):
            soname = os.path.basename(dep.name).split('.')[0]
            sharedLibraryRequires.add(soname)
            if soname.startswith('lib'):
                sharedLibraryRequires.add(soname[3:])
            else:
                sharedLibraryRequires.add('lib%s' %soname)
        troveLibraries = set()
        for path in pathMap.iterkeys():
            basename = os.path.basename(path)
            if basename.startswith('lib') and basename.find('.') >= 0:
                troveLibraries.add(basename[3:].split('.')[0])

        self.recipe.synchronizeLogs()
        f = file(self.recipe.getSubscribeLogPath())

        libRe = re.compile('^-l[a-zA-Z]+$')
        libDirRe = re.compile('^-L/..*$')

        def logLineTokens():
            for logLine in f:
                logLine = logLine.strip()
                if not self.r.match(logLine):
                    continue
                yield logLine.split()
            for logLine in self.logLines:
                yield logLine.split()

        def pathSetToTroveSet(pathSet):
            troveSet = set()
            for path in pathSet:
                for pathReq in set(trove.getName()
                                   for trove in db.iterTrovesByPath(path)):
                    pathReqCandidates = _providesNames(pathReq)
                    # remove any recursive or non-existing buildreqs
                    pathReqCandidates = [x for x in pathReqCandidates 
                                         if db.hasTroveByName(x)]
                    if not pathReqCandidates:
                        continue
                    allPossibleProviders.update(pathReqCandidates)
                    # only the best option
                    pathReqCandidates = pathReqCandidates[0:1]
                    # now apply exceptions
                    pathReqCandidates = self._removeExceptionsFromList(
                        pathReqCandidates)
                    troveSet.add(pathReqCandidates[0])
            return troveSet

        def buildDirContains(libName):
            # If we can find this library built somewhere in the
            # builddir, chances are that the internal library is
            # what is being linked to in any case.
            if self.buildDirLibNames is None:
                # walk builddir once, the first time this is called
                self.buildDirLibNames = set()
                for dirpath, dirnames, filenames in os.walk(builddir):
                    for fileName in filenames:
                        if fileName.startswith('lib') and '.' in fileName:
                            self.buildDirLibNames.add(fileName[3:].split('.')[0])
            return libName in self.buildDirLibNames

        for tokens in logLineTokens():
            libNames = set(x[2:] for x in tokens if libRe.match(x))
            # Add to this set, for this line only, system library dirs,
            # nothing in destdir or builddir
            libDirs = self.libDirs.copy()
            for libDir in set(x[2:].rstrip('/') for x in tokens
                              if libDirRe.match(x) and
                                 not x[2:].startswith(destdir) and
                                 not x[2:].startswith(builddir)):
                libDir = util.normpath(libDir)
                libDirs.setdefault(util.normpath('%s%s' %(cfg.root, libDir)), libDir)
                libDirs.setdefault(libDir, libDir)
            for libName in sorted(list(libNames)):
                if libName not in foundLibNames:
                    if libName in sharedLibraryRequires:
                        foundLibNames.add(libName)
                        continue
                    if libName in troveLibraries:
                        foundLibNames.add(libName)
                        continue
                    if buildDirContains(libName):
                        foundLibNames.add(libName)
                        continue

                    foundLibs = set()
                    for libDirRoot, libDir in libDirs.iteritems():
                        for ext in ('a', 'so'):
                            # If there is no .a, look for the .so in case
                            # no shared library dependency is found from
                            # packaged files (CNP-132)
                            if util.exists('%s/lib%s.%s' %(libDirRoot, libName, ext)):
                                foundLibs.add('%s/lib%s.%s' %(libDir, libName, ext))
                                break
                    troveSet = pathSetToTroveSet(foundLibs)

                    if len(troveSet) == 1:
                        # found just one, we can confidently recommend it
                        recommended = list(troveSet)[0]
                        if recommended not in transitiveBuildRequires:
                            self.info("Add '%s' to buildRequires for -l%s (%s)",
                                      recommended, libName,
                                      ', '.join(sorted(list(foundLibs))))
                            missingBuildRequires.add(recommended)
                            foundLibNames.add(libName)

                    elif len(troveSet):
                        # found more, we might need to recommend a choice
                        tooManyChoices.setdefault(libName, [
                                  ' '.join(sorted(list(foundLibs))),
                                  "', '".join(sorted(list(troveSet)))])

                    elif foundLibs:
                        # found files on system, but no troves providing them
                        noTroveFound.setdefault(libName,
                                  ' '.join(sorted(list(foundLibs))))
                        
                    else:
                        # note that this does not prevent us from
                        # *looking* again, because the next time
                        # there might be a useful -L in the link line
                        noLibraryFound.add(libName)
                            
        if tooManyChoices:
            for libName in sorted(list(tooManyChoices.keys())):
                if libName not in foundLibNames:
                    # Found multiple choices for libName, and never came
                    # up with a better recommendation, so recommend a choice.
                    # Note: perhaps someday this can become an error
                    # when we have a better sense of how frequently
                    # it is wrong...
                    foundLibNames.add(libName)
                    foundLibs, troveSet = tooManyChoices[libName]
                    self.warn('Multiple troves match files %s for -l%s:'
                              ' choose one of the following entries'
                              " for buildRequires: '%s'",
                              foundLibs, libName, troveSet)

        if noTroveFound:
            for libName in sorted(list(noTroveFound.keys())):
                if libName not in foundLibNames:
                    # Never found any trove containing these libraries,
                    # not even a file in the builddir
                    foundLibNames.add(libName)
                    foundLibs = noTroveFound[libName]
                    self.info('No trove found matching any of files'
                              ' %s for -l%s:'
                              ' possible missing buildRequires',
                              foundLibs, libName)

        if noLibraryFound:
            for libName in sorted(list(noLibraryFound)):
                if libName not in foundLibNames:
                    # Note: perhaps someday this can become an error
                    # when we have a better sense of how frequently
                    # it is wrong...
                    self.info('No files found matching -l%s:'
                              ' possible missing buildRequires', libName)

        if missingBuildRequires:
            self.talk('add to buildRequires: %s',
                       str(sorted(list(missingBuildRequires))))
            reportMissingBuildRequires(self.recipe, missingBuildRequires)

        if allPossibleProviders:
            reportFoundBuildRequires(self.recipe, allPossibleProviders)

        f.close()
Exemplo n.º 40
0
    def install(self, flags, troveCs):
        ACTION_RESTORE = 1
        ACTION_SKIP = 2
        ACTION_CONFLICT = 3

        rc = SingleCapsuleOperation.install(self, flags, troveCs)
        if rc is None:
            # parent class thinks we should just ignore this troveCs; I'm
            # not going to argue with it (it's probably because the capsule
            # hasn't changed
            return None

        (oldTrv, trv) = rc
        trvInfo = troveCs.getNewNameVersionFlavor()
        oldTrvInfo = troveCs.getOldNameVersionFlavor()
        hasCapsule = troveCs.hasCapsule()

        # Updates the fsJob metadata for installing the current trove.
        # It assumes files are replaced on install, and complains if something
        # is in the way unless the appropriate flags are set. This is a very
        # much simplified version of FilesystemJob._singleTrove() which maps
        # out a complete install strategy for native packages. Note that
        # we walk all of the files in this trove, not just the new files
        # or the changed files, because RPM installs all of the files.
        toRestore = []

        changedByPathId = dict((x[0], x) for x in troveCs.getChangedFileList())

        # things which aren't change, new, or removed are unchanged
        unchangedByPathId = (set(x[0] for x in trv.iterFileList()) -
                             set(changedByPathId.iterkeys()) -
                             set(x[0] for x in troveCs.getNewFileList()) -
                             set(troveCs.getOldFileList()))

        l = []
        for oldFileInfo in troveCs.getChangedFileList():
            oldFileId, oldVersion = oldTrv.getFile(oldFileInfo[0])[1:3]
            l.append((oldFileInfo[0], oldFileId, oldVersion))

        for unchangedPathId in unchangedByPathId:
            unchangedFileId, unchangedFileVersion = \
                                    trv.getFile(unchangedPathId)[1:3]
            l.append((unchangedPathId, unchangedFileId, unchangedFileVersion))

        fileObjs = self.db.getFileVersions(l)
        fileObjsByPathId = dict([(x[0], y)
                                 for x, y in itertools.izip(l, fileObjs)])

        for fileInfo in trv.iterFileList():
            pathId, path, fileId, version = fileInfo

            if os.path.dirname(path) in self.netSharedPath:
                # we do nothing. really. nothing.
                #
                # we don't back it up. we don't mark it as removed in
                # our database. we don't look for conflicts. nothing.
                continue

            if pathId in changedByPathId:
                oldFileId = oldTrv.getFile(pathId)[1]
                fileChange = self.changeSet.getFileChange(oldFileId, fileId)
                if (oldFileId == fileId):
                    # only the version number changed; we don't need
                    # to merge anything here
                    fileObj = fileObjsByPathId[pathId]
                elif fileChange[0] == '\x01':
                    fileObj = fileObjsByPathId[pathId]
                    fileObj.twm(fileChange, fileObj)
                else:
                    fileObj = files.ThawFile(fileChange, pathId)
            elif pathId in unchangedByPathId:
                fileObj = fileObjsByPathId[pathId]
            else:
                # if it's not changed and it's not unchanged, it must be new
                fileStream = self.changeSet.getFileChange(None, fileId)
                fileObj = files.ThawFile(fileStream, pathId)

            absolutePath = util.joinPaths(self.root, path)

            if (fileObj.flags.isCapsuleAddition()):
                # this was added to the package outside of the RPM; we don't
                # have any responsibility for it
                continue
            elif (trove.conaryContents(hasCapsule, pathId, fileObj)
                  and fileObj.lsTag != 'd'):
                # this content isn't part of the capsule; remember to put
                # it back when RPM is done
                self.preservePath(path, unlink=True)
                continue

            s = util.lstat(absolutePath)
            if not s:
                # there is nothing in the way, so there is nothing which
                # concerns us here. Track the file for later.
                toRestore.append((fileInfo, fileObj))
                continue

            action = ACTION_CONFLICT

            existingOwners = list(
                self.db.iterFindPathReferences(path,
                                               justPresent=True,
                                               withStream=True))

            if existingOwners:
                # Don't complain about files owned by the previous version
                # of this trove.
                l = [x for x in existingOwners if x[0:3] == oldTrvInfo]
                if l:
                    existingOwners.remove(l[0])

                if not existingOwners:
                    action = ACTION_RESTORE
            elif stat.S_ISDIR(s.st_mode) and fileObj.lsTag == 'd':
                # Don't let existing directories stop us from taking over
                # ownership of the directory
                action = ACTION_RESTORE
            elif fileObj.flags.isInitialContents():
                # Initial contents files may be restored on top of things
                # already in the filesystem. They're ghosts or config files
                # and RPM will get the contents right either way, and we
                # should remove them either way.
                action = ACTION_RESTORE

            if action == ACTION_CONFLICT and not existingOwners:
                # Check for "conflicts" that might just be a view across a
                # symlink.
                if self.fsJob.findAliasedRemovals(absolutePath):
                    action = ACTION_RESTORE

            if action == ACTION_CONFLICT and existingOwners:
                if fileId in [x[4] for x in existingOwners]:
                    # The files share metadata same. Whatever it looks like on
                    # disk, RPM is going to blow it away with the new one.
                    for info in existingOwners:
                        self.fsJob.sharedFile(info[0], info[1], info[2],
                                              info[3])
                    action = ACTION_RESTORE
                elif path.startswith('/usr/share/doc/'):
                    # Mirror badness Red Hat patches into RPM for rhel4
                    # and rhel5
                    action = ACTION_RESTORE
                else:
                    existingFiles = [
                        files.ThawFile(x[5], pathId) for x in existingOwners
                    ]

                    compatibility = [
                        1 for x in existingFiles if fileObj.compatibleWith(x)
                    ]

                    if 1 in compatibility:
                        # files can be shared even though the fileId's
                        # are different
                        for info in existingOwners:
                            self.fsJob.sharedFile(info[0], info[1], info[2],
                                                  info[3])
                        action = ACTION_RESTORE
                    elif 1 in [
                            files.rpmFileColorCmp(x, fileObj)
                            for x in existingFiles
                    ]:
                        # rpm file colors and the default rpm setting for
                        # file color policy make elf64 files silently replace
                        # elf32 files. follow that behavior here.
                        #
                        # no, i'm not making this up
                        #
                        # yes, really
                        action = ACTION_SKIP
                    elif (self._checkReplaceManagedFiles(flags, path) or 1 in [
                            files.rpmFileColorCmp(fileObj, x)
                            for x in existingFiles
                    ]):
                        # The files are different. Bail unless we're supposed
                        # to replace managed files.
                        existingFile = files.FileFromFilesystem(
                            absolutePath, pathId)
                        for info in existingOwners:
                            self.fsJob.userRemoval(
                                fileObj=existingFile,
                                content=filecontents.FromFilesystem(
                                    absolutePath),
                                *info[0:4])
                        action = ACTION_RESTORE
                    else:
                        # it's not up to us to decide if this is a true
                        # conflict; the database layer will do that for
                        # us (see checkPathConflicts)
                        action = ACTION_RESTORE
            elif flags.replaceUnmanagedFiles:
                # we don't own it, but it's on disk. RPM will just write over
                # it and we have the flag saying we're good with that
                action = ACTION_RESTORE

            if action == ACTION_RESTORE:
                # We may proceed, and RPM will replace this file for us. We
                # need to track that it's being restored to avoid conflicts
                # with other restorations though.
                toRestore.append((fileInfo, fileObj))
            elif action == ACTION_CONFLICT:
                # The file exists already, we can't share it, and we're not
                # allowed to overwrite it.
                self._error(
                    errors.FileInWayError(util.normpath(path),
                                          troveCs.getName(),
                                          troveCs.getNewVersion(),
                                          troveCs.getNewFlavor()))
            else:
                assert (action == ACTION_SKIP)
                self.preservePath(path, unlink=False)
                self.fsJob.userRemoval(trv.getName(), trv.getVersion(),
                                       trv.getFlavor(), pathId)

        # toRestore is the list of what is going to be restored. We need to get
        # the fileObjects which will be created so we can track them in the
        # filesystem job. This lets the filesystem job look for resolveable
        # conflicts within this update. We handle newly created files first
        # and files which have changed (so we have to look up the diff)
        # a bit later.
        for fileInfo, fileObj in toRestore:
            fullPath = util.joinPaths(self.root, path)
            self.fsJob._restore(fileObj,
                                fullPath,
                                trvInfo,
                                "restoring %s from RPM",
                                restoreFile=False,
                                fileId=fileId)
Exemplo n.º 41
0
def getTrovesToDisplay(db,
                       troveSpecs,
                       pathList=[],
                       whatProvidesList=[],
                       exactFlavors=False):
    """ Finds the given trove and path specifiers, and returns matching
        (n,v,f) tuples.
        @param db: database to search
        @type db: local.database.Database
        @param troveSpecs: troves to search for
        @type troveSpecs: list of troveSpecs (n[=v][[f]])
        @param pathList: paths which should be linked to some trove in this
                         database.
        @type pathList: list of strings
        @param whatProvidesList: deps to search for providers of
        @type whatProvidesList: list of strings

        @raises TroveSpecError: Raised if one of the troveSpecs is of an
                                invalid format

        @note: This function calls database routines which could raise any
               errors defined in L{dbstore.sqlerrors}

        @rtype: troveTupleList (list of (name, version, flavor) tuples),
                and a boolean that stats whether the troves returned should
                be considered primary (and therefore not compressed ever).
    """

    primary = True

    if troveSpecs:
        troveSpecs = [ cmdline.parseTroveSpec(x, allowEmptyName=False) \
                                                        for x in troveSpecs ]
    else:
        troveSpecs = []

    normPathList = [
        util.realpath(os.path.abspath(util.normpath(x))) for x in pathList
    ]

    troveTups = []
    for path, origPath in itertools.izip(normPathList, pathList):
        if origPath.endswith('/'):
            allPaths = [path + '/' + x for x in os.listdir(db.root + path)]
        else:
            allPaths = [path]

        for thisPath in allPaths:
            for trove in db.iterTrovesByPath(thisPath):
                troveTups.append(
                    (trove.getName(), trove.getVersion(), trove.getFlavor()))

    if whatProvidesList:
        results = db.getTrovesWithProvides(whatProvidesList)
        troveTups.extend(itertools.chain(*results.itervalues()))

    if not (troveSpecs or pathList or whatProvidesList):
        troveTups = sorted(db.iterAllTroves())
        primary = False
    else:
        results = db.findTroves(None, troveSpecs, exactFlavors=exactFlavors)

        for troveSpec in troveSpecs:
            troveTups.extend(results.get(troveSpec, []))

    return troveTups, primary
Exemplo n.º 42
0
    def addPluggableRequirements(self, path, fullpath, pkgFiles, macros):
        if hasattr(self.recipe, '_getCapsulePathsForFile'):
            if self.recipe._getCapsulePathsForFile(path):
                # since capsules do not convert to relative symlinks,
                # we cannot depend on getting the realpath.  Unless
                # we resolve that, assume that capsule-provided
                # dependencies will be sufficient for pkgconfig files.
                return

        # parse pkgconfig file
        variables = {}
        requirements = set()
        libDirs = []
        libraries = set()
        variableLineRe = re.compile('^[a-zA-Z0-9]+=')
        filesRequired = []

        pcContents = [x.strip() for x in file(fullpath).readlines()]
        for pcLine in pcContents:
            # interpolate variables: assume variables are interpreted
            # line-by-line while processing
            pcLineIter = pcLine
            while True:
                for var in variables:
                    pcLineIter = pcLineIter.replace(var, variables[var])
                if pcLine == pcLineIter:
                    break
                pcLine = pcLineIter
            pcLine = pcLineIter

            if variableLineRe.match(pcLine):
                key, val = pcLine.split('=', 1)
                variables['${%s}' % key] = val
            else:
                if (pcLine.startswith('Requires')
                        or pcLine.startswith('Lib')) and ':' in pcLine:
                    keyWord, args = pcLine.split(':', 1)
                    # split on ',' and ' '
                    argList = itertools.chain(
                        *[x.split(',') for x in args.split()])
                    argList = [x for x in argList if x]
                    if keyWord.startswith('Requires'):
                        versionNext = False
                        for req in argList:
                            if [x for x in '<=>' if x in req]:
                                versionNext = True
                                continue
                            if versionNext:
                                versionNext = False
                                continue
                            requirements.add(req)
                    elif keyWord.startswith('Lib'):
                        for lib in argList:
                            if lib.startswith('-L'):
                                libDirs.append(lib[2:])
                            elif lib.startswith('-l'):
                                libraries.add(lib[2:])
                            else:
                                pass

        # find referenced pkgconfig files and add requirements
        for req in requirements:
            candidateFileNames = [
                '%(destdir)s%(libdir)s/pkgconfig/' + req + '.pc',
                '%(destdir)s%(datadir)s/pkgconfig/' + req + '.pc',
                '%(libdir)s/pkgconfig/' + req + '.pc',
                '%(datadir)s/pkgconfig/' + req + '.pc',
            ]
            candidateFileNames = [x % macros for x in candidateFileNames]
            candidateFiles = [util.exists(x) for x in candidateFileNames]
            if True in candidateFiles:
                filesRequired.append(
                    (candidateFileNames[candidateFiles.index(True)],
                     'pkg-config'))
            else:
                self.warn('pkg-config file %s.pc not found', req)
                continue

        # find referenced library files and add requirements
        libraryPaths = sorted(list(self.systemLibPaths))
        for libDir in libDirs:
            if libDir not in libraryPaths:
                libraryPaths.append(libDir)
        for library in libraries:
            found = False
            for libDir in libraryPaths:
                candidateFileNames = [
                    macros.destdir + libDir + '/lib' + library + '.so',
                    macros.destdir + libDir + '/lib' + library + '.a',
                    libDir + '/lib' + library + '.so',
                    libDir + '/lib' + library + '.a',
                ]
                candidateFiles = [util.exists(x) for x in candidateFileNames]
                if True in candidateFiles:
                    filesRequired.append(
                        (candidateFileNames[candidateFiles.index(True)],
                         'library'))
                    found = True
                    break

            if not found:
                self.warn('library file lib%s not found', library)
                continue

        for fileRequired, fileType in filesRequired:
            if fileRequired.startswith(macros.destdir):
                # find requirement in packaging
                fileRequired = util.normpath(os.path.realpath(fileRequired))
                fileRequired = fileRequired[
                    len(util.normpath(os.path.realpath(macros.destdir))):]
                autopkg = self.recipe.autopkg
                troveName = autopkg.componentMap[fileRequired].name
                package, component = troveName.split(':', 1)
                if component in ('devellib', 'lib'):
                    for preferredComponent in ('devel', 'devellib'):
                        develTroveName = ':'.join(
                            (package, preferredComponent))
                        if develTroveName in autopkg.components and autopkg.components[
                                develTroveName]:
                            # found a non-empty :devel compoment
                            troveName = develTroveName
                            break
                self._addRequirement(path, troveName, [], pkgFiles,
                                     deps.TroveDependencies)
            else:
                troveName = self._enforceProvidedPath(fileRequired,
                                                      fileType=fileType,
                                                      unmanagedError=True)
                if troveName:
                    self._addRequirement(path, troveName, [], pkgFiles,
                                         deps.TroveDependencies)