def iter(self, dir, file, root): try: dirs = file.getFullName().split('/') except: dirs = [] # Don't process non-source code. for skip in ['.git', 'build-']: if skip in dirs: return for d in dirs: if d.find(skip) == 0: return try: fin = open(file.getFullName(), 'r') except IOError: return tmp = tempfile.mktemp() fout = open(tmp, 'w') state = 0 # 0 - outside block, 1 - inside block blocks = 0 # number of copyright blocks found try: lines = fin.readlines() except: lines = [] for line in lines: pos = line.find(self.pattern[state]) if pos >= 0: state = state ^ 1 if not state: blocks += 1 else: prefix = line[0:pos] suffix = line[pos + len(self.pattern[0]):] fout.write(line) for text in self.copyright: fout.write('%s%s%s' % (prefix, text, suffix)) if not state: fout.write(line) fin.close() fout.close() # Commit the replaced text only if a complete copyright # block was found. This will make sure a single copyright # tag in a file does not cause all the code to be lost. if blocks: print(file.getFullName()) shutil.copymode(file.getFullName(), tmp) try: shutil.copyfile(tmp, file.getFullName()) except IOError as msg: pass os.unlink(tmp)
def builder(self, path, file, root): if not root: root = self.dist.getReleasePath() dir = os.path.join(root, path) fullname = os.path.join(dir, file.getName()) if file.getFullName() == fullname: return if not os.path.isdir(dir): os.makedirs(dir) # Create the new distribution either with all symbolic links # into the mirror, contrib, and local rpms. Or copy # everything. The idea is local distributions should be all # symlinks, but a published base distribution (like the NPACI # Rocks master) should be copys. This keeps the FTP chroot # environment happy, extends the lifetime of the release past # that of scattered RPMS. It may also make sense to have your # master distribution for your cluster done by copy. if self.useLinks: file.symlink(fullname, self.dist.getRootPath()) else: # For copied distributions, the timestamps of the new # files are forced to that of the source files. This # keeps wget happy. if os.path.islink(file.getFullName()): os.symlink(os.readlink(file.getFullName()), fullname) else: shutil.copy(file.getFullName(), fullname) os.utime(fullname, (file.getTimestamp(), file.getTimestamp()))
def iter(self, dir, file, root): try: dirs = file.getFullName().split('/') except: dirs = [ ] # Don't process non-source code. for skip in [ '.git', 'build-' ]: if skip in dirs: return for d in dirs: if d.find(skip) == 0: return try: fin = open(file.getFullName(), 'r') except IOError: return tmp = tempfile.mktemp() fout = open(tmp, 'w') state = 0 # 0 - outside block, 1 - inside block blocks = 0 # number of copyright blocks found for line in fin.readlines(): pos = string.find(line, self.pattern[state]) if pos >= 0: state = state ^ 1 if not state: blocks += 1 else: prefix = line[0:pos] suffix = line[pos+len(self.pattern[0]):] fout.write(line) for text in self.copyright: fout.write('%s%s%s' % ( prefix, text, suffix)) if not state: fout.write(line) fin.close() fout.close() # Commit the replaced text only if a complete copyright # block was found. This will make sure a single copyright # tag in a file does not cause all the code to be lost. if blocks: print(file.getFullName()) shutil.copymode(file.getFullName(), tmp) try: shutil.copyfile(tmp, file.getFullName()) except IOError, msg: pass
def getCDInfoFromXML(self): retval = (None, None, None) self.mountCD() cdtree = stack.file.Tree('/mnt/cdrom') for dir in cdtree.getDirs(): for file in cdtree.getFiles(dir): try: xmlfile = stack.file.RollInfoFile( file.getFullName()) rollname = xmlfile.getRollName() rollversion = xmlfile.getRollVersion() rollarch = xmlfile.getRollArch() if rollname != None and \ rollversion != None and \ rollarch != None: retval = (rollname, \ rollversion, rollarch) break except: continue if retval != (None, None, None): break return retval
def buildKickstart(self): print 'Installing XML Kickstart profiles' build = self.dist.getBuildPath() for rpm in self.dist.getRPMS(): tok = rpm.getBaseName().split('-') if tok[0] != 'roll': continue try: k = tok.index('kickstart') rollname = '-'.join(tok[1:k]) except ValueError: continue print '\tusing %s' % rpm.getBaseName() self.applyRPM(rpm.getBaseName(), build) # Copy local profiles into the distribution. if self.withSiteProfiles: print ' installing "site" profiles...' tree = self.dist.getSiteProfilesTree() for dir in tree.getDirs(): for file in tree.getFiles(dir): path = os.path.join(build, dir) if not os.path.isdir(path): os.makedirs(path) shutil.copy(file.getFullName(), os.path.join(path, file.getName())) # make sure apache can read site XML file.chmod(0664)
def getRPMS(self, path): """Return a list of all the RPMs in the given path, if multiple versions of a package are found only the most recent one will be included""" dict = {} tree = stack.file.Tree(os.path.join(os.getcwd(), path)) for dir in tree.getDirs(): for file in tree.getFiles(dir): try: file.getPackageName() except AttributeError: continue # skip all non-rpm files # Skip RPMS for other architecures if file.getPackageArch() not in self.getCPUs(): continue # Resolve package versions if newest == True: name = file.getUniqueName() else: name = file.getFullName() if not dict.has_key(name) or file >= dict[name]: dict[name] = file # convert the dictionary to a list and return all the RPMFiles list = [] for e in dict.keys(): list.append(dict[e]) return list
def getRPMS(self, path): """Return a list of all the RPMs in the given path, if multiple versions of a package are found only the most recent one will be included""" dict = {} tree = stack.file.Tree(os.path.join(os.getcwd(), path)) for dir in tree.getDirs(): for file in tree.getFiles(dir): try: file.getPackageName() except: continue # skip all non-rpm files # Skip RPMS for other architecures if file.getPackageArch() not in self.getCPUs(): continue # Resolve package versions if newest is True: name = file.getUniqueName() else: name = file.getFullName() if (name not in dict) or (file >= dict[name]): dict[name] = file # convert the dictionary to a list and return all the RPMFiles list = [] for e in dict.keys(): list.append(dict[e]) return list
def buildKickstart(self): print('Installing XML Kickstart profiles') build = self.dist.getBuildPath() for rpm in self.dist.getRPMS(): tok = rpm.getBaseName().split('-') if tok[0] != 'roll': continue try: k = tok.index('kickstart') rollname = '-'.join(tok[1:k]) except ValueError: continue print('\tinstalling %s' % rpm.getBaseName()) self.applyRPM(rpm.getBaseName(), build) # Copy local profiles into the distribution. if self.withSiteProfiles: print('\tinstalling site-profiles') tree = self.dist.getSiteProfilesTree() for dir in tree.getDirs(): for file in tree.getFiles(dir): path = os.path.join(build, dir) if not os.path.isdir(path): os.makedirs(path) shutil.copy(file.getFullName(), os.path.join(path, file.getName())) # make sure apache can read site XML file.chmod(0o664) # Copy cart profiles into the distribution. for cart in self.carts: print('\tinstalling %s cart' % cart) tree = stack.file.Tree(os.path.join('/export/stack/carts', cart)) for file in tree.getFiles('graph'): shutil.copy(file.getFullName(), os.path.join(build, 'graphs', 'default', file.getName())) for file in tree.getFiles('nodes'): shutil.copy(file.getFullName(), os.path.join(build, 'nodes', file.getName()))
def buildKickstart(self): print 'Installing XML Kickstart profiles' build = self.dist.getBuildPath() for rpm in self.dist.getRPMS(): tok = rpm.getBaseName().split('-') if tok[0] != 'roll': continue try: k = tok.index('kickstart') rollname = '-'.join(tok[1:k]) except ValueError: continue print '\tinstalling %s' % rpm.getBaseName() self.applyRPM(rpm.getBaseName(), build) # Copy local profiles into the distribution. if self.withSiteProfiles: print '\tinstalling site-profiles' tree = self.dist.getSiteProfilesTree() for dir in tree.getDirs(): for file in tree.getFiles(dir): path = os.path.join(build, dir) if not os.path.isdir(path): os.makedirs(path) shutil.copy(file.getFullName(), os.path.join(path, file.getName())) # make sure apache can read site XML file.chmod(0664) # Copy cart profiles into the distribution. for cart in self.carts: print '\tinstalling %s cart' % cart tree = stack.file.Tree(os.path.join('/export/stack/carts', cart)) for file in tree.getFiles('graph'): shutil.copy( file.getFullName(), os.path.join(build, 'graphs', 'default', file.getName())) for file in tree.getFiles('nodes'): shutil.copy(file.getFullName(), os.path.join(build, 'nodes', file.getName()))
def copyFile(self, path, file, root): if file.getName() in ['TRANS.TBL']: return dir = os.path.join(root, path) fullname = os.path.join(dir, file.getName()) if not os.path.isdir(dir): os.makedirs(dir) shutil.copy(file.getFullName(), fullname) os.utime(fullname, (file.getTimestamp(), file.getTimestamp()))
def normalizer(self, path, file, root): if not root: root = self.dist.getReleasePath() dir = os.path.join(root, path) fullname = os.path.join(dir, file.getName()) # Reset the File to represent the one we just created in the new # distribution. if file.getFullName() != fullname: file.setFile(fullname)
def copyFile(self, path, file, root): if file.getName() in [ 'TRANS.TBL' ]: return dir = os.path.join(root, path) fullname = os.path.join(dir, file.getName()) if not os.path.isdir(dir): os.makedirs(dir) shutil.copy(file.getFullName(), fullname) os.utime(fullname, (file.getTimestamp(), file.getTimestamp()))
def iter(self, dir, file, root): try: # # skip source control files # if '.git' in file.getFullName().split('/'): return except: pass try: fin = open(file.getFullName(), 'r') except IOError: return tmp = tempfile.mktemp() fout = open(tmp, 'w') state = 0 # 0 - outside block, 1 - inside block blocks = 0 # number of copyright blocks found for line in fin.readlines(): pos = string.find(line, self.pattern[state]) if pos >= 0: state = state ^ 1 if not state: blocks += 1 else: prefix = line[0:pos] suffix = line[pos + len(self.pattern[0]):] fout.write(line) for text in self.copyright: fout.write('%s%s%s' % (prefix, text, suffix)) if not state: fout.write(line) fin.close() fout.close() # Commit the replaced text only if a complete copyright # block was found. This will make sure a single copyright # tag in a file does not cause all the code to be lost. if blocks: print file.getFullName() shutil.copymode(file.getFullName(), tmp) try: shutil.copyfile(tmp, file.getFullName()) except IOError, msg: pass
def run(self, params, args): (dryrun, ) = self.fillParams([('dryrun', 'true')]) dryrun = self.str2bool(dryrun) script = [] script.append('#!/bin/sh') rolls = [] for roll in args: rolls.append(roll) if sys.stdin.isatty(): xml = self.command( 'list.host.xml', ['localhost', 'pallet=%s' % string.join(rolls, ',')]) else: xml = sys.stdin.read() reader = Sax2.Reader() gen = getattr(stack.gen, 'Generator_%s' % self.os)() gen.parse(xml) distPath = os.path.join( self.command('report.distribution')[:-1], 'default') tree = stack.file.Tree(distPath) rpm_list = {} for file in tree.getFiles(os.path.join(self.arch, 'RedHat', 'RPMS')): if isinstance(file, stack.file.RPMFile): rpm_list[file.getBaseName()] = \ file.getFullName() rpm_list["%s.%s" % (file.getBaseName(), \ file.getPackageArch())] = \ file.getFullName() rpms = [] for line in gen.generate('packages'): if line.find('%package') == -1 and line.find('%end') == -1: rpms.append(line) for rpm in rpms: if rpm in rpm_list.keys(): script.append('yum install -y %s' % rpm) script.append(rpm_force_template % rpm_list[rpm]) cur_proc = False for line in gen.generate('post'): if not line.startswith('%post') and not line.startswith('%end'): script.append(line) else: if cur_proc == True: script.append('__POSTEOF__') script.append('%s %s' % (interpreter, t_name)) cur_proc = False try: i = line.split().index('--interpreter') except ValueError: continue interpreter = line.split()[i + 1] t_name = tempfile.mktemp() cur_proc = True script.append('cat > %s << "__POSTEOF__"' % t_name) script.append('\n# Boot scripts\n') for line in gen.generate('boot'): # # skip lines that start with '%post' or '%end' # if line[0:5] == '%post' or line[0:4] == '%end': continue script.append(line) if dryrun: self.addText(string.join(script, '\n')) else: os.system(string.join(script, '\n'))
def run(self, hosts): pathSource = os.path.join(os.sep, 'export', 'stack', 'salt') self.pathCompiled = os.path.join(pathSource, 'compiled') attrs = {} for host in hosts['hosts']: attrs[host] = {} host_path = os.path.join(self.pathCompiled, host) if os.path.exists(host_path): shutil.rmtree(host_path) for row in self.owner.call('list.host.attr', hosts['hosts']): attrs[row['host']][row['attr']] = row['value'] for host in hosts['hosts']: host_attrs = attrs[host] paths = [os.path.join(pathSource, 'default')] if 'environment' in host_attrs: paths.append( os.path.join(pathSource, host_attrs['environment'])) for path in paths: tree = stack.file.Tree(path) for dir in tree.getDirs(): for file in tree.getFiles(dir): if not os.path.splitext(file.getName())[1] == '.xml': continue parser = make_parser() handler = FileHandler(host_attrs) parser.setContentHandler(handler) parser.feed(handler.getXMLHeader()) parser.feed('<salt>') fin = open(file.getFullName(), 'r') # Parse as XML-ish to pickup the # entities that correspond to host # attributes. But we assume the <file # ...> and </file> tags start at col 0 # on a line by themselves. Anything # between the file tags is first # sanitized for '<' and '&' # (non-entity refs only). This keep # the file in XML-ish mode, and # prevents the need for using CDATA. # Node.xml files do not behave this # we, so while a node.xml <file> # section can move into this framework # the reverse is not true (yet). for line in fin.readlines(): if line.find('<file') == 0 or \ line.find('</file>') == 0: s = line else: s = '' for i in range(0, len(line)): if line[i] == '<': s += '<' elif line[i] == '&': if not re.match( '&[A-Za-z_.][A-Za-z0-9_.]*;', line[i:]): s += '&' else: s += line[i] else: s += line[i] try: parser.feed(s) except: print('Parsing Error file %s line %s' % (file.getFullName(), line)) parser.feed('</salt>') fin.close() for parsed in handler.getFiles(): if stack.cond.EvalCondExpr(parsed.get('cond'), host_attrs): self.writeSalt(host, parsed)
def run(self, params, args): (dryrun, ) = self.fillParams([ ('dryrun', 'true')]) dryrun = self.str2bool(dryrun) script = [] script.append('#!/bin/sh') rolls = [] for roll in args: rolls.append(roll) if sys.stdin.isatty(): xml = self.command('list.host.xml', [ 'localhost', 'pallet=%s' % string.join(rolls, ',') ]) else: xml = sys.stdin.read() reader = Sax2.Reader() gen = getattr(stack.gen,'Generator_%s' % self.os)() gen.parse(xml) distPath = os.path.join(self.command('report.distribution')[:-1], 'default') tree = stack.file.Tree(distPath) rpm_list = {} for file in tree.getFiles(os.path.join(self.arch, 'RedHat', 'RPMS')): if isinstance(file, stack.file.RPMFile): rpm_list[file.getBaseName()] = \ file.getFullName() rpm_list["%s.%s" % (file.getBaseName(), \ file.getPackageArch())] = \ file.getFullName() rpms = [] for line in gen.generate('packages'): if line.find('%package') == -1 and line.find('%end') == -1: rpms.append(line) for rpm in rpms: if rpm in rpm_list.keys(): script.append('yum install -y %s' % rpm) script.append(rpm_force_template % rpm_list[rpm]) cur_proc = False for line in gen.generate('post'): if not line.startswith('%post') and not line.startswith('%end'): script.append(line) else: if cur_proc == True: script.append('__POSTEOF__') script.append('%s %s' % (interpreter, t_name)) cur_proc = False try: i = line.split().index('--interpreter') except ValueError: continue interpreter = line.split()[i+1] t_name = tempfile.mktemp() cur_proc = True script.append('cat > %s << "__POSTEOF__"' % t_name) script.append('\n# Boot scripts\n') for line in gen.generate('boot'): # # skip lines that start with '%post' or '%end' # if line[0:5] == '%post' or line[0:4] == '%end': continue script.append(line) if dryrun: self.addText(string.join(script, '\n')) else: os.system(string.join(script, '\n'))
def run(self, args): (clean, prefix, treeinfo) = args name = None vers = None arch = None file = open(treeinfo, 'r') for line in file.readlines(): a = line.split('=') if len(a) != 2: continue key = a[0].strip() value = a[1].strip() if key == 'family': if value == 'Red Hat Enterprise Linux': name = 'RHEL' elif value == 'CentOS': name = 'CentOS' elif key == 'version': vers = value elif key == 'arch': arch = value file.close() if not name: name = "BaseOS" if not vers: vers = stack.version if not arch: arch = 'x86_64' OS = 'redhat' roll_dir = os.path.join(prefix, name, vers, OS, arch) destdir = os.path.join(roll_dir, 'RPMS') if clean and os.path.exists(roll_dir): print 'Cleaning %s version %s ' % (name, vers), print 'for %s from pallets directory' % arch os.system('/bin/rm -rf %s' % roll_dir) os.makedirs(roll_dir) print 'Copying "%s" (%s,%s) pallet ...' % (name, vers, arch) if not os.path.exists(destdir): os.makedirs(destdir) cdtree = stack.file.Tree(self.owner.mountPoint) for dir in cdtree.getDirs(): for file in cdtree.getFiles(dir): if not file.getName().endswith('.rpm'): continue if file.getPackageArch() != 'src' and \ file.getBaseName() != 'comps' and \ file.getName() != 'comps.rpm' \ and not (arch == 'i386' and \ re.match('^kudzu.*', file.getBaseName())): os.system('cp -p %s %s' % ( file.getFullName(), destdir)) # # lay down a minimal roll XML config file # f = open('%s' % os.path.join(roll_dir, 'roll-%s.xml' % name), 'w') f.write('<roll name="%s" interface="6.0.2">\n' % name) f.write('<info version="%s" release="%s" arch="%s" os="%s"/>\n' % (vers, stack.release, arch, OS)) f.write('<iso maxsize="0" bootable="0"/>\n') f.write('</roll>\n') f.close() return (name, vers, arch)
def run(self, args): import stack (clean, prefix, treeinfo) = args name = None vers = None arch = None file = open(treeinfo, 'r') for line in file.readlines(): a = line.split('=') if len(a) != 2: continue key = a[0].strip() value = a[1].strip() if key == 'family': if value == 'Red Hat Enterprise Linux': name = 'RHEL' elif value == 'CentOS': name = 'CentOS' elif key == 'version': vers = value elif key == 'arch': arch = value file.close() if not name: name = "BaseOS" if not vers: vers = stack.version if not arch: arch = 'x86_64' OS = 'redhat' roll_dir = os.path.join(prefix, name, vers, OS, arch) destdir = os.path.join(roll_dir, 'RPMS') print 'stack.release (%s)' % stack.release if stack.release == '7.x': liveosdir = os.path.join(roll_dir, 'LiveOS') if clean and os.path.exists(roll_dir): print 'Cleaning %s version %s ' % (name, vers), print 'for %s from pallets directory' % arch os.system('/bin/rm -rf %s' % roll_dir) os.makedirs(roll_dir) print 'Copying "%s" (%s,%s) pallet ...' % (name, vers, arch) if not os.path.exists(destdir): os.makedirs(destdir) if stack.release == '7.x' and not os.path.exists(liveosdir): os.makedirs(liveosdir) cdtree = stack.file.Tree(self.owner.mountPoint) for dir in cdtree.getDirs(): for file in cdtree.getFiles(dir): if stack.release == '7.x' and dir == 'LiveOS' \ and file.getName().endswith('.img'): os.system('cp -p %s %s' % ( file.getFullName(), liveosdir)) else: if not file.getName().endswith('.rpm'): continue if file.getPackageArch() != 'src' and \ file.getBaseName() != 'comps' and \ file.getName() != 'comps.rpm' \ and not (arch == 'i386' and \ re.match('^kudzu.*', file.getBaseName())): os.system('cp -p %s %s' % ( file.getFullName(), destdir)) # # lay down a minimal roll XML config file # f = open('%s' % os.path.join(roll_dir, 'roll-%s.xml' % name), 'w') f.write('<roll name="%s" interface="6.0.2">\n' % name) f.write('<info version="%s" release="%s" arch="%s" os="%s"/>\n' % (vers, stack.release, arch, OS)) f.write('<iso maxsize="0" bootable="0"/>\n') f.write('</roll>\n') f.close() return (name, vers, arch)