def doFile(self, path): if hasattr(self.recipe, '_getCapsulePathsForFile'): if self.recipe._getCapsulePathsForFile(path): return d = self.macros.destdir fullpath = util.joinPaths(d, path) if not (os.path.isfile(fullpath) and util.isregular(fullpath)): return f = file(fullpath) lines = f.readlines() f.close() foundChkconfig = False for line in lines: line = line.strip() if not line: continue if not line.startswith('#'): # chkconfig tag must come before any uncommented lines break if line.find('chkconfig:') != -1: foundChkconfig = True break if line.find('### BEGIN INIT INFO') != -1: foundChkconfig = True break if not foundChkconfig: self.warn( "initscript %s must contain chkconfig information before any uncommented lines", path)
def doFile(self, path): if hasattr(self.recipe, '_getCapsulePathsForFile'): if self.recipe._getCapsulePathsForFile(path): return d = self.macros.destdir fullpath = util.joinPaths(d, path) if not (os.path.isfile(fullpath) and util.isregular(fullpath)): return f = file(fullpath) lines = f.readlines() f.close() foundChkconfig = False for line in lines: line = line.strip() if not line: continue if not line.startswith('#'): # chkconfig tag must come before any uncommented lines break if line.find('chkconfig:') != -1: foundChkconfig = True break if line.find('### BEGIN INIT INFO') != -1: foundChkconfig = True break if not foundChkconfig: self.warn("initscript %s must contain chkconfig information before any uncommented lines", path)
def _sosymlink(self, dirname, names): section = os.path.basename(dirname) for name in names: path = dirname + os.sep + name if util.isregular(path): # if only .so, change to symlink f = file(path) lines = f.readlines(512) # we really don't need the whole file f.close() # delete comment lines first newlines = [] for line in lines: # newline means len(line) will be at least 1 if len(line) > 1 and not self.commentexp.search(line[:-1]): newlines.append(line) lines = newlines # now see if we have only a .so line to replace # only replace .so with symlink if the file exists # in order to deal with searchpaths if len(lines) == 1: line = lines[0] # remove newline and other trailing whitespace if it exists line = line.rstrip() match = self.soexp.search(line) if match: matchlist = match.group(1).split('/') l = len(matchlist) if l == 1 or matchlist[l-2] == section: # no directory specified, or in the same # directory: targetpath = os.sep.join((dirname, matchlist[l-1])) if (os.path.exists(targetpath) and os.path.isfile(targetpath)): self.info('replacing %s (%s) with symlink %s', name, match.group(0), os.path.basename(match.group(1))) os.remove(path) os.symlink(os.path.basename(match.group(1)), path) else: # either the canonical .so manN/foo.N or an # absolute path /usr/share/man/manN/foo.N # .so is relative to %(mandir)s and the other # man page is in a different dir, so add ../ target = "../%s/%s" %(matchlist[l-2], matchlist[l-1]) targetpath = os.sep.join((dirname, target)) if os.path.exists(targetpath): self.info('replacing %s (%s) with symlink %s', name, match.group(0), target) os.remove(path) os.symlink(target, path)
def doFile(self, filename): if hasattr(self.recipe, '_getCapsulePathsForFile'): if self.recipe._getCapsulePathsForFile(filename): return fullpath = self.macros.destdir + filename if os.path.isfile(fullpath) and util.isregular(fullpath): m = self.recipe.magic[filename] if m and m.name == 'ELF' and 'soname' in m.contents: self.info(filename) self.recipe.autopkg.pathMap[filename].tags.set("shlib")
def _compress(self, dirname, names): for name in names: path = dirname + os.sep + name if util.isregular(path): if not self.gzip: self.gzip = self._findProg('gzip') util.execute('gzip -f -n -9 ' + dirname + os.sep + name) try: self.recipe.recordMove(dirname + os.sep + name, dirname + os.sep + name + '.gz') except AttributeError: pass
def _uncompress(self, dirname, names): for name in names: path = dirname + os.sep + name if name.endswith('.gz') and util.isregular(path): if not self.gunzip: self.gunzip = self._findProg('gunzip') util.execute('gunzip ' + dirname + os.sep + name) try: self.recipe.recordMove(util.joinPaths(dirname, name), util.joinPaths(dirname, name)[:-3]) except AttributeError: pass if name.endswith('.bz2') and util.isregular(path): if not self.bunzip: self.bunzip = self._findProg('bunzip2') util.execute('bunzip2 ' + dirname + os.sep + name) try: self.recipe.recordMove(util.joinPaths(dirname, name), util.joinPaths(dirname, name)[:-4]) except AttributeError: pass
def doFile(self, filename): fullpath = self.macros.destdir + filename if os.path.isfile(fullpath) and util.isregular(fullpath): recipe = self.recipe for pkg in self.recipe.autopkg.findComponents(filename): f = pkg.getFile(filename) # config or initialContents wins in capsule packages if f.flags.isConfig() or f.flags.isInitialContents(): return self.info(filename) for pkg in self.recipe.autopkg.findComponents(filename): f = pkg.getFile(filename) f.flags.isTransient(True)
def doFile(self, filename): source = util.joinPaths(self.builddir, filename) dest = util.joinPaths(self.destdir, filename) if os.path.exists(dest): return if not util.isregular(source): # will not be a directory, but might be a symlink or something return util.mkdirChain(os.path.dirname(dest)) shutil.copy2(source, dest) os.chmod(dest, 0644) # this file should not be counted as making package non-empty self.recipe._autoCreatedFileCount += 1
def doFile(self, path): if hasattr(self.recipe, '_getCapsulePathsForFile'): if self.recipe._getCapsulePathsForFile(path): return fullpath = util.joinPaths(self.macros['destdir'], path) if not util.isregular(fullpath): return mode = os.lstat(fullpath)[stat.ST_MODE] if mode & 0111: # has some executable bit set return self.warn('non-executable library %s, changing to mode 0755', path) os.chmod(fullpath, 0755)
def doFile(self, filename): fullpath = self.macros.destdir + filename hardlinkMap = self.recipe.autopkg.findComponent(filename).hardlinkMap if (os.path.isfile(fullpath) and util.isregular(fullpath) and not self._fileIsBinary(filename, fullpath, maxsize=20*1024) and not filename in hardlinkMap): self.info(filename) for pkg in self.recipe.autopkg.findComponents(filename): f = pkg.getFile(filename) f.flags.isConfig(True) else: # RPM config files are handled more like initialcontents, # so for for files that conary can't be sure it can display # diffs on, we should make them be initialcontents for # conary verify purposes self.recipe.InitialContents(filename)
def doFile(self, filename): fullpath = self.macros.destdir + filename hardlinkMap = self.recipe.autopkg.findComponent(filename).hardlinkMap if (os.path.isfile(fullpath) and util.isregular(fullpath) and not self._fileIsBinary(filename, fullpath, maxsize=20 * 1024) and not filename in hardlinkMap): self.info(filename) for pkg in self.recipe.autopkg.findComponents(filename): f = pkg.getFile(filename) f.flags.isConfig(True) else: # RPM config files are handled more like initialcontents, # so for for files that conary can't be sure it can display # diffs on, we should make them be initialcontents for # conary verify purposes self.recipe.InitialContents(filename)
def _compress(self, dirname, names): for name in names: path = dirname + os.sep + name # Resolve symlinks before comporessing to make sure the target # file gets compressed. if os.path.islink(path): path = os.path.realpath(path) # Assumed already compressed. if path.endswith('.gz'): continue # Already been compressed via symlink lookup or the dangling # symlink policy will catch it. if not os.path.exists(path): continue if util.isregular(path): if not self.gzip: self.gzip = self._findProg('gzip') util.execute(self.gzip + ' -f -n -9 ' + path) try: self.recipe.recordMove(path, path + '.gz') except AttributeError: pass
def loadPolicy(recipeObj, policySet = None, internalPolicyModules = (), basePolicy = Policy): # path -> module policyPathMap = {} # bucket -> ordered list of policy objects policies = { TESTSUITE: [], DESTDIR_PREPARATION: [], DESTDIR_MODIFICATION: [], PACKAGE_CREATION: [], PACKAGE_MODIFICATION: [], ENFORCEMENT: [], ERROR_REPORTING: [], GROUP_ENFORCEMENT: [], } # bucket -> dict of policy classes policyMap = dict((b, {}) for b in policies.keys()) # name -> policy classes policyNameMap = {} # Load conary internal policy. Note that these are now loaded first, # so that pluggable policy may override internal policy. Also note # that the order of the internalPolicyModules list matters, as the # *last* module of a given name which is loaded wins. This is # also true of pluggable policy, so the policyDirs config option must # be carefully constructed in reverse order of preference. import conary.build.destdirpolicy import conary.build.derivedpolicy import conary.build.derivedcapsulepolicy import conary.build.infopolicy import conary.build.packagepolicy import conary.build.capsulepolicy import conary.build.grouppolicy for pt in internalPolicyModules: m = sys.modules['conary.build.'+pt] for symbolName in m.__dict__.keys(): policyCls = m.__dict__[symbolName] if type(policyCls) is not classType: continue if symbolName[0] != '_' and issubclass(policyCls, basePolicy): policyNameMap[symbolName] = policyCls # Load pluggable policy for policyDir in recipeObj.cfg.policyDirs: if not os.path.isdir(policyDir): continue for filename in sorted(os.listdir(policyDir)): fullpath = os.sep.join((policyDir, filename)) if not filename.endswith('.py') or not util.isregular(fullpath): continue # do not load shared libraries as policy! desc = [x for x in imp.get_suffixes() if x[0] == '.py'][0] f = file(fullpath) modname = filename[:-3] m = imp.load_module(modname, f, fullpath, desc) f.close() policyPathMap[fullpath] = m for symbolName in m.__dict__: policyCls = m.__dict__[symbolName] if type(policyCls) is not classType: continue if policySet is not None and symbolName not in policySet: continue if symbolName[0].isupper() and issubclass(policyCls, basePolicy): policyNameMap[symbolName] = policyCls # Enforce dependencies missingDeps = [] for policyCls in policyNameMap.values(): if hasattr(policyCls, 'requires'): for reqName, reqType in policyCls.requires: if reqType & REQUIRED and reqName not in policyNameMap: missingDeps.append((policyCls.__name__, reqName)) if missingDeps: raise PolicyError, '\n'.join( ('policy %s missing required policy %s' %(x,y) for x, y in missingDeps)) # Sort and initialize for policyName, policyCls in policyNameMap.iteritems(): policyMap[policyCls.bucket][policyName]=policyNameMap[policyName] for bucket in policyMap.keys(): dg = graph.DirectedGraph() for policyCls in policyMap[bucket].values(): dg.addNode(policyCls) if hasattr(policyCls, 'requires'): for reqName, reqType in policyCls.requires: if reqType & ORDERED and reqName in policyMap[bucket]: if reqType & PRIOR: dg.addEdge(policyNameMap[reqName], policyCls) else: dg.addEdge(policyCls, policyNameMap[reqName]) # test for dependency loops depLoops = [x for x in dg.getStronglyConnectedComponents() if len(x) > 1] if depLoops: # convert to names depLoops = [sorted(x.__name__ for x in y) for y in depLoops] raise PolicyError, '\n'.join( 'found dependency loop: %s' %', '.join(y) for y in depLoops) # store an ordered list of initialized policy objects policies[bucket] = [x(recipeObj) for x in dg.getTotalOrdering( nodeSort=lambda a, b: cmp(a[1].__name__, b[1].__name__))] return policyPathMap, policies
def loadPolicy(recipeObj, policySet=None, internalPolicyModules=(), basePolicy=Policy): # path -> module policyPathMap = {} # bucket -> ordered list of policy objects policies = { TESTSUITE: [], DESTDIR_PREPARATION: [], DESTDIR_MODIFICATION: [], PACKAGE_CREATION: [], PACKAGE_MODIFICATION: [], ENFORCEMENT: [], ERROR_REPORTING: [], GROUP_ENFORCEMENT: [], } # bucket -> dict of policy classes policyMap = dict((b, {}) for b in policies.keys()) # name -> policy classes policyNameMap = {} # Load conary internal policy. Note that these are now loaded first, # so that pluggable policy may override internal policy. Also note # that the order of the internalPolicyModules list matters, as the # *last* module of a given name which is loaded wins. This is # also true of pluggable policy, so the policyDirs config option must # be carefully constructed in reverse order of preference. import conary.build.destdirpolicy import conary.build.derivedpolicy import conary.build.derivedcapsulepolicy import conary.build.infopolicy import conary.build.packagepolicy import conary.build.capsulepolicy import conary.build.grouppolicy for pt in internalPolicyModules: m = sys.modules['conary.build.' + pt] for symbolName in m.__dict__.keys(): policyCls = m.__dict__[symbolName] if type(policyCls) is not classType: continue if symbolName[0] != '_' and issubclass(policyCls, basePolicy): policyNameMap[symbolName] = policyCls # Load pluggable policy for policyDir in recipeObj.cfg.policyDirs: if not os.path.isdir(policyDir): continue for filename in sorted(os.listdir(policyDir)): fullpath = os.sep.join((policyDir, filename)) if not filename.endswith('.py') or not util.isregular(fullpath): continue # do not load shared libraries as policy! desc = [x for x in imp.get_suffixes() if x[0] == '.py'][0] f = file(fullpath) modname = filename[:-3] m = imp.load_module(modname, f, fullpath, desc) f.close() policyPathMap[fullpath] = m for symbolName in m.__dict__: policyCls = m.__dict__[symbolName] if type(policyCls) is not classType: continue if policySet is not None and symbolName not in policySet: continue if symbolName[0].isupper() and issubclass( policyCls, basePolicy): policyNameMap[symbolName] = policyCls # Enforce dependencies missingDeps = [] for policyCls in policyNameMap.values(): if hasattr(policyCls, 'requires'): for reqName, reqType in policyCls.requires: if reqType & REQUIRED and reqName not in policyNameMap: missingDeps.append((policyCls.__name__, reqName)) if missingDeps: raise PolicyError, '\n'.join( ('policy %s missing required policy %s' % (x, y) for x, y in missingDeps)) # Sort and initialize for policyName, policyCls in policyNameMap.iteritems(): policyMap[policyCls.bucket][policyName] = policyNameMap[policyName] for bucket in policyMap.keys(): dg = graph.DirectedGraph() for policyCls in policyMap[bucket].values(): dg.addNode(policyCls) if hasattr(policyCls, 'requires'): for reqName, reqType in policyCls.requires: if reqType & ORDERED and reqName in policyMap[bucket]: if reqType & PRIOR: dg.addEdge(policyNameMap[reqName], policyCls) else: dg.addEdge(policyCls, policyNameMap[reqName]) # test for dependency loops depLoops = [ x for x in dg.getStronglyConnectedComponents() if len(x) > 1 ] if depLoops: # convert to names depLoops = [sorted(x.__name__ for x in y) for y in depLoops] raise PolicyError, '\n'.join( 'found dependency loop: %s' % ', '.join(y) for y in depLoops) # store an ordered list of initialized policy objects policies[bucket] = [ x(recipeObj) for x in dg.getTotalOrdering( nodeSort=lambda a, b: cmp(a[1].__name__, b[1].__name__)) ] return policyPathMap, policies
def magic(path, basedir=''): """ Returns a magic class with information about the file mentioned """ if basedir and not basedir.endswith('/'): basedir += '/' n = basedir + path if not util.exists(n) or not util.isregular(n): return None oldmode = None mode = os.lstat(n)[stat.ST_MODE] if (mode & 0400) != 0400: oldmode = mode os.chmod(n, mode | 0400) f = file(n) if oldmode is not None: os.chmod(n, oldmode) b = f.read(4096) f.close() if len(b) > 4 and b[0] == '\x7f' and b[1:4] == "ELF": return ELF(path, basedir, b) elif len(b) > 14 and b[0:14] == '!<arch>\ndebian': return deb(path, basedir) elif len(b) > 7 and b[0:7] == "!<arch>": return ar(path, basedir, b) elif len(b) > 2 and b[0] == '\x1f' and b[1] == '\x8b': try: uncompressedBuffer = gzip_module.GzipFile(n).read(4096) if _tarMagic(uncompressedBuffer): return tar_gz(path, basedir, b, uncompressedBuffer) except (IOError, zlib.error): # gzip sometimes raises IOError instead of any module-specific # errors; in either error case just do not consider this a # gzip file. # Note that gzip or tar_gz magic does not imply that the # entire file has been tested to have no compression errors! pass return gzip(path, basedir, b) elif len(b) > 3 and b[0:3] == "BZh": try: uncompressedBuffer = bz2.BZ2File(n).read(4096) if _tarMagic(uncompressedBuffer): return tar_bz2(path, basedir, b, uncompressedBuffer) except IOError: # bz2 raises IOError instead of any module specific errors pass return bzip(path, basedir, b) elif len(b) > 6 and b[0:6] == "\xFD\x37\x7A\x58\x5A\x00": # http://tukaani.org/xz/xz-file-format.txt return xz(path, basedir, b) elif len(b) > 4 and b[0:4] == "\xEA\x3F\x81\xBB": return changeset(path, basedir, b) elif len(b) > 4 and b[0:4] == "PK\x03\x04": # Zip file. Peek inside the file to extract the file list try: zf = zipfile.ZipFile(n) namelist = set(i.filename for i in zf.infolist() if not i.filename.endswith('/') and i.file_size > 0) except (IOError, zipfile.BadZipfile): # zipfile raises IOError on some malformed zip files # We are producing a dummy jar or ZIP with no contents if path.endswith('.jar'): return jar(path, basedir) return ZIP(path, basedir) except RuntimeError: # not a proper zip archive -- likely a .car archive CNY-2871 namelist = None if namelist is not None: if 'META-INF/application.xml' in namelist: return EAR(path, basedir, zipFileObj=zf, fileList=namelist) elif 'WEB-INF/web.xml' in namelist: return WAR(path, basedir, zipFileObj=zf, fileList=namelist) elif 'META-INF/MANIFEST.MF' in namelist: return jar(path, basedir, zipFileObj=zf, fileList=namelist) #elif path.endswith('.par'): # perl archive else: return ZIP(path, basedir, zipFileObj=zf, fileList=namelist) elif _javaMagic(b): return java(path, basedir, b) elif len(b) > 4 and b[0:2] == "#!": if b.find( '# This wrapper script should never be moved out of the build directory.\n' '# If it is, it will not operate correctly.') > 0: return ltwrapper(path, basedir, b) return script(path, basedir, _line(b)) elif (len(b) > 130 and b[0:2] == 'MZ' and b[78:117] == "This program cannot be run in DOS mode." and b[128:130] == "PE"): # FIXME - this is not sufficient to detect a CIL file this # will match all PE executables. See ECMA-335, partition ii, # section 25 return CIL(path, basedir, b) elif (len(b) > 4 and b[:4] == "\xed\xab\xee\xdb"): return RPM(path, basedir) elif (len(b) > len(MSI_MAGIC_STRINGS[0]) and [x for x in MSI_MAGIC_STRINGS if b[:len(x)] == x]): return MSI(path, basedir) elif len(b) > len(WIM_MAGIC_STRING) and \ b[:len(WIM_MAGIC_STRING)] == WIM_MAGIC_STRING: return WIM(path, basedir) elif _tarMagic(b): return tar(path, basedir, b) return None
def magic(path, basedir=''): """ Returns a magic class with information about the file mentioned """ if basedir and not basedir.endswith('/'): basedir += '/' n = basedir+path if not util.exists(n) or not util.isregular(n): return None oldmode = None mode = os.lstat(n)[stat.ST_MODE] if (mode & 0400) != 0400: oldmode = mode os.chmod(n, mode | 0400) f = file(n) if oldmode is not None: os.chmod(n, oldmode) b = f.read(4096) f.close() if len(b) > 4 and b[0] == '\x7f' and b[1:4] == "ELF": return ELF(path, basedir, b) elif len(b) > 14 and b[0:14] == '!<arch>\ndebian': return deb(path, basedir) elif len(b) > 7 and b[0:7] == "!<arch>": return ar(path, basedir, b) elif len(b) > 2 and b[0] == '\x1f' and b[1] == '\x8b': try: uncompressedBuffer = gzip_module.GzipFile(n).read(4096) if _tarMagic(uncompressedBuffer): return tar_gz(path, basedir, b, uncompressedBuffer) except (IOError, zlib.error): # gzip sometimes raises IOError instead of any module-specific # errors; in either error case just do not consider this a # gzip file. # Note that gzip or tar_gz magic does not imply that the # entire file has been tested to have no compression errors! pass return gzip(path, basedir, b) elif len(b) > 3 and b[0:3] == "BZh": try: uncompressedBuffer = bz2.BZ2File(n).read(4096) if _tarMagic(uncompressedBuffer): return tar_bz2(path, basedir, b, uncompressedBuffer) except IOError: # bz2 raises IOError instead of any module specific errors pass return bzip(path, basedir, b) elif len(b) > 6 and b[0:6] == "\xFD\x37\x7A\x58\x5A\x00": # http://tukaani.org/xz/xz-file-format.txt return xz(path, basedir, b) elif len(b) > 4 and b[0:4] == "\xEA\x3F\x81\xBB": return changeset(path, basedir, b) elif len(b) > 4 and b[0:4] == "PK\x03\x04": # Zip file. Peek inside the file to extract the file list try: zf = zipfile.ZipFile(n) namelist = set(i.filename for i in zf.infolist() if not i.filename.endswith('/') and i.file_size > 0) except (IOError, zipfile.BadZipfile): # zipfile raises IOError on some malformed zip files # We are producing a dummy jar or ZIP with no contents if path.endswith('.jar'): return jar(path, basedir) return ZIP(path, basedir) except RuntimeError: # not a proper zip archive -- likely a .car archive CNY-2871 namelist = None if namelist is not None: if 'META-INF/application.xml' in namelist: return EAR(path, basedir, zipFileObj = zf, fileList = namelist) elif 'WEB-INF/web.xml' in namelist: return WAR(path, basedir, zipFileObj = zf, fileList = namelist) elif 'META-INF/MANIFEST.MF' in namelist: return jar(path, basedir, zipFileObj = zf, fileList = namelist) #elif path.endswith('.par'): # perl archive else: return ZIP(path, basedir, zipFileObj = zf, fileList = namelist) elif _javaMagic(b): return java(path, basedir, b) elif len(b) > 4 and b[0:2] == "#!": if b.find( '# This wrapper script should never be moved out of the build directory.\n' '# If it is, it will not operate correctly.') > 0: return ltwrapper(path, basedir, b) return script(path, basedir, _line(b)) elif (len(b) > 130 and b[0:2] == 'MZ' and b[78:117] == "This program cannot be run in DOS mode." and b[128:130] == "PE"): # FIXME - this is not sufficient to detect a CIL file this # will match all PE executables. See ECMA-335, partition ii, # section 25 return CIL(path, basedir, b) elif (len(b) > 4 and b[:4] == "\xed\xab\xee\xdb"): return RPM(path, basedir) elif (len(b) > len(MSI_MAGIC_STRINGS[0]) and [ x for x in MSI_MAGIC_STRINGS if b[:len(x)] == x ]): return MSI(path,basedir) elif len(b) > len(WIM_MAGIC_STRING) and \ b[:len(WIM_MAGIC_STRING)] == WIM_MAGIC_STRING: return WIM(path,basedir) elif _tarMagic(b): return tar(path, basedir, b) return None