def iter(self, dir, file, root):
try:
dirs = file.getFullName().split('/')
except:
dirs = []
# Don't process non-source code.
for skip in ['.git', 'build-']:
if skip in dirs:
return
for d in dirs:
if d.find(skip) == 0:
return
try:
fin = open(file.getFullName(), 'r')
except IOError:
return
tmp = tempfile.mktemp()
fout = open(tmp, 'w')
state = 0 # 0 - outside block, 1 - inside block
blocks = 0 # number of copyright blocks found
try:
lines = fin.readlines()
except:
lines = []
for line in lines:
pos = line.find(self.pattern[state])
if pos >= 0:
state = state ^ 1
if not state:
blocks += 1
else:
prefix = line[0:pos]
suffix = line[pos + len(self.pattern[0]):]
fout.write(line)
for text in self.copyright:
fout.write('%s%s%s' % (prefix, text, suffix))
if not state:
fout.write(line)
fin.close()
fout.close()
# Commit the replaced text only if a complete copyright
# block was found. This will make sure a single copyright
# tag in a file does not cause all the code to be lost.
if blocks:
print(file.getFullName())
shutil.copymode(file.getFullName(), tmp)
try:
shutil.copyfile(tmp, file.getFullName())
except IOError as msg:
pass
os.unlink(tmp)
def builder(self, path, file, root):
if not root:
root = self.dist.getReleasePath()
dir = os.path.join(root, path)
fullname = os.path.join(dir, file.getName())
if file.getFullName() == fullname:
return
if not os.path.isdir(dir):
os.makedirs(dir)
# Create the new distribution either with all symbolic links
# into the mirror, contrib, and local rpms. Or copy
# everything. The idea is local distributions should be all
# symlinks, but a published base distribution (like the NPACI
# Rocks master) should be copys. This keeps the FTP chroot
# environment happy, extends the lifetime of the release past
# that of scattered RPMS. It may also make sense to have your
# master distribution for your cluster done by copy.
if self.useLinks:
file.symlink(fullname, self.dist.getRootPath())
else:
# For copied distributions, the timestamps of the new
# files are forced to that of the source files. This
# keeps wget happy.
if os.path.islink(file.getFullName()):
os.symlink(os.readlink(file.getFullName()), fullname)
else:
shutil.copy(file.getFullName(), fullname)
os.utime(fullname, (file.getTimestamp(), file.getTimestamp()))
def builder(self, path, file, root):
if not root:
root = self.dist.getReleasePath()
dir = os.path.join(root, path)
fullname = os.path.join(dir, file.getName())
if file.getFullName() == fullname:
return
if not os.path.isdir(dir):
os.makedirs(dir)
# Create the new distribution either with all symbolic links
# into the mirror, contrib, and local rpms. Or copy
# everything. The idea is local distributions should be all
# symlinks, but a published base distribution (like the NPACI
# Rocks master) should be copys. This keeps the FTP chroot
# environment happy, extends the lifetime of the release past
# that of scattered RPMS. It may also make sense to have your
# master distribution for your cluster done by copy.
if self.useLinks:
file.symlink(fullname, self.dist.getRootPath())
else:
# For copied distributions, the timestamps of the new
# files are forced to that of the source files. This
# keeps wget happy.
if os.path.islink(file.getFullName()):
os.symlink(os.readlink(file.getFullName()), fullname)
else:
shutil.copy(file.getFullName(), fullname)
os.utime(fullname, (file.getTimestamp(), file.getTimestamp()))
def iter(self, dir, file, root):
try:
dirs = file.getFullName().split('/')
except:
dirs = [ ]
# Don't process non-source code.
for skip in [ '.git', 'build-' ]:
if skip in dirs:
return
for d in dirs:
if d.find(skip) == 0:
return
try:
fin = open(file.getFullName(), 'r')
except IOError:
return
tmp = tempfile.mktemp()
fout = open(tmp, 'w')
state = 0 # 0 - outside block, 1 - inside block
blocks = 0 # number of copyright blocks found
for line in fin.readlines():
pos = string.find(line, self.pattern[state])
if pos >= 0:
state = state ^ 1
if not state:
blocks += 1
else:
prefix = line[0:pos]
suffix = line[pos+len(self.pattern[0]):]
fout.write(line)
for text in self.copyright:
fout.write('%s%s%s' % (
prefix,
text,
suffix))
if not state:
fout.write(line)
fin.close()
fout.close()
# Commit the replaced text only if a complete copyright
# block was found. This will make sure a single copyright
# tag in a file does not cause all the code to be lost.
if blocks:
print(file.getFullName())
shutil.copymode(file.getFullName(), tmp)
try:
shutil.copyfile(tmp, file.getFullName())
except IOError, msg:
pass
def getCDInfoFromXML(self):
retval = (None, None, None)
self.mountCD()
cdtree = stack.file.Tree('/mnt/cdrom')
for dir in cdtree.getDirs():
for file in cdtree.getFiles(dir):
try:
xmlfile = stack.file.RollInfoFile(
file.getFullName())
rollname = xmlfile.getRollName()
rollversion = xmlfile.getRollVersion()
rollarch = xmlfile.getRollArch()
if rollname != None and \
rollversion != None and \
rollarch != None:
retval = (rollname, \
rollversion, rollarch)
break
except:
continue
if retval != (None, None, None):
break
return retval
def buildKickstart(self):
print 'Installing XML Kickstart profiles'
build = self.dist.getBuildPath()
for rpm in self.dist.getRPMS():
tok = rpm.getBaseName().split('-')
if tok[0] != 'roll':
continue
try:
k = tok.index('kickstart')
rollname = '-'.join(tok[1:k])
except ValueError:
continue
print '\tusing %s' % rpm.getBaseName()
self.applyRPM(rpm.getBaseName(), build)
# Copy local profiles into the distribution.
if self.withSiteProfiles:
print ' installing "site" profiles...'
tree = self.dist.getSiteProfilesTree()
for dir in tree.getDirs():
for file in tree.getFiles(dir):
path = os.path.join(build, dir)
if not os.path.isdir(path):
os.makedirs(path)
shutil.copy(file.getFullName(),
os.path.join(path, file.getName()))
# make sure apache can read site XML
file.chmod(0664)
def getRPMS(self, path):
"""Return a list of all the RPMs in the given path, if multiple
versions of a package are found only the most recent one will
be included"""
dict = {}
tree = stack.file.Tree(os.path.join(os.getcwd(), path))
for dir in tree.getDirs():
for file in tree.getFiles(dir):
try:
file.getPackageName()
except AttributeError:
continue # skip all non-rpm files
# Skip RPMS for other architecures
if file.getPackageArch() not in self.getCPUs():
continue
# Resolve package versions
if newest == True:
name = file.getUniqueName()
else:
name = file.getFullName()
if not dict.has_key(name) or file >= dict[name]:
dict[name] = file
# convert the dictionary to a list and return all the RPMFiles
list = []
for e in dict.keys():
list.append(dict[e])
return list
def getCDInfoFromXML(self):
retval = (None, None, None)
self.mountCD()
cdtree = stack.file.Tree('/mnt/cdrom')
for dir in cdtree.getDirs():
for file in cdtree.getFiles(dir):
try:
xmlfile = stack.file.RollInfoFile(
file.getFullName())
rollname = xmlfile.getRollName()
rollversion = xmlfile.getRollVersion()
rollarch = xmlfile.getRollArch()
if rollname != None and \
rollversion != None and \
rollarch != None:
retval = (rollname, \
rollversion, rollarch)
break
except:
continue
if retval != (None, None, None):
break
return retval
def getRPMS(self, path):
"""Return a list of all the RPMs in the given path, if multiple
versions of a package are found only the most recent one will
be included"""
dict = {}
tree = stack.file.Tree(os.path.join(os.getcwd(), path))
for dir in tree.getDirs():
for file in tree.getFiles(dir):
try:
file.getPackageName()
except:
continue # skip all non-rpm files
# Skip RPMS for other architecures
if file.getPackageArch() not in self.getCPUs():
continue
# Resolve package versions
if newest is True:
name = file.getUniqueName()
else:
name = file.getFullName()
if (name not in dict) or (file >= dict[name]):
dict[name] = file
# convert the dictionary to a list and return all the RPMFiles
list = []
for e in dict.keys():
list.append(dict[e])
return list
def buildKickstart(self):
print('Installing XML Kickstart profiles')
build = self.dist.getBuildPath()
for rpm in self.dist.getRPMS():
tok = rpm.getBaseName().split('-')
if tok[0] != 'roll':
continue
try:
k = tok.index('kickstart')
rollname = '-'.join(tok[1:k])
except ValueError:
continue
print('\tinstalling %s' % rpm.getBaseName())
self.applyRPM(rpm.getBaseName(), build)
# Copy local profiles into the distribution.
if self.withSiteProfiles:
print('\tinstalling site-profiles')
tree = self.dist.getSiteProfilesTree()
for dir in tree.getDirs():
for file in tree.getFiles(dir):
path = os.path.join(build, dir)
if not os.path.isdir(path):
os.makedirs(path)
shutil.copy(file.getFullName(),
os.path.join(path, file.getName()))
# make sure apache can read site XML
file.chmod(0o664)
# Copy cart profiles into the distribution.
for cart in self.carts:
print('\tinstalling %s cart' % cart)
tree = stack.file.Tree(os.path.join('/export/stack/carts',
cart))
for file in tree.getFiles('graph'):
shutil.copy(file.getFullName(),
os.path.join(build, 'graphs', 'default', file.getName()))
for file in tree.getFiles('nodes'):
shutil.copy(file.getFullName(),
os.path.join(build, 'nodes', file.getName()))
def buildKickstart(self):
print 'Installing XML Kickstart profiles'
build = self.dist.getBuildPath()
for rpm in self.dist.getRPMS():
tok = rpm.getBaseName().split('-')
if tok[0] != 'roll':
continue
try:
k = tok.index('kickstart')
rollname = '-'.join(tok[1:k])
except ValueError:
continue
print '\tinstalling %s' % rpm.getBaseName()
self.applyRPM(rpm.getBaseName(), build)
# Copy local profiles into the distribution.
if self.withSiteProfiles:
print '\tinstalling site-profiles'
tree = self.dist.getSiteProfilesTree()
for dir in tree.getDirs():
for file in tree.getFiles(dir):
path = os.path.join(build, dir)
if not os.path.isdir(path):
os.makedirs(path)
shutil.copy(file.getFullName(),
os.path.join(path, file.getName()))
# make sure apache can read site XML
file.chmod(0664)
# Copy cart profiles into the distribution.
for cart in self.carts:
print '\tinstalling %s cart' % cart
tree = stack.file.Tree(os.path.join('/export/stack/carts', cart))
for file in tree.getFiles('graph'):
shutil.copy(
file.getFullName(),
os.path.join(build, 'graphs', 'default', file.getName()))
for file in tree.getFiles('nodes'):
shutil.copy(file.getFullName(),
os.path.join(build, 'nodes', file.getName()))
def copyFile(self, path, file, root):
if file.getName() in ['TRANS.TBL']:
return
dir = os.path.join(root, path)
fullname = os.path.join(dir, file.getName())
if not os.path.isdir(dir):
os.makedirs(dir)
shutil.copy(file.getFullName(), fullname)
os.utime(fullname, (file.getTimestamp(), file.getTimestamp()))
def normalizer(self, path, file, root):
if not root:
root = self.dist.getReleasePath()
dir = os.path.join(root, path)
fullname = os.path.join(dir, file.getName())
# Reset the File to represent the one we just created in the new
# distribution.
if file.getFullName() != fullname:
file.setFile(fullname)
def copyFile(self, path, file, root): if file.getName() in [ 'TRANS.TBL' ]: return dir = os.path.join(root, path) fullname = os.path.join(dir, file.getName()) if not os.path.isdir(dir): os.makedirs(dir) shutil.copy(file.getFullName(), fullname) os.utime(fullname, (file.getTimestamp(), file.getTimestamp()))
def normalizer(self, path, file, root):
if not root:
root = self.dist.getReleasePath()
dir = os.path.join(root, path)
fullname = os.path.join(dir, file.getName())
# Reset the File to represent the one we just created in the new
# distribution.
if file.getFullName() != fullname:
file.setFile(fullname)
def iter(self, dir, file, root):
try:
#
# skip source control files
#
if '.git' in file.getFullName().split('/'):
return
except:
pass
try:
fin = open(file.getFullName(), 'r')
except IOError:
return
tmp = tempfile.mktemp()
fout = open(tmp, 'w')
state = 0 # 0 - outside block, 1 - inside block
blocks = 0 # number of copyright blocks found
for line in fin.readlines():
pos = string.find(line, self.pattern[state])
if pos >= 0:
state = state ^ 1
if not state:
blocks += 1
else:
prefix = line[0:pos]
suffix = line[pos + len(self.pattern[0]):]
fout.write(line)
for text in self.copyright:
fout.write('%s%s%s' % (prefix, text, suffix))
if not state:
fout.write(line)
fin.close()
fout.close()
# Commit the replaced text only if a complete copyright
# block was found. This will make sure a single copyright
# tag in a file does not cause all the code to be lost.
if blocks:
print file.getFullName()
shutil.copymode(file.getFullName(), tmp)
try:
shutil.copyfile(tmp, file.getFullName())
except IOError, msg:
pass
def run(self, params, args):
(dryrun, ) = self.fillParams([('dryrun', 'true')])
dryrun = self.str2bool(dryrun)
script = []
script.append('#!/bin/sh')
rolls = []
for roll in args:
rolls.append(roll)
if sys.stdin.isatty():
xml = self.command(
'list.host.xml',
['localhost',
'pallet=%s' % string.join(rolls, ',')])
else:
xml = sys.stdin.read()
reader = Sax2.Reader()
gen = getattr(stack.gen, 'Generator_%s' % self.os)()
gen.parse(xml)
distPath = os.path.join(
self.command('report.distribution')[:-1], 'default')
tree = stack.file.Tree(distPath)
rpm_list = {}
for file in tree.getFiles(os.path.join(self.arch, 'RedHat', 'RPMS')):
if isinstance(file, stack.file.RPMFile):
rpm_list[file.getBaseName()] = \
file.getFullName()
rpm_list["%s.%s" % (file.getBaseName(), \
file.getPackageArch())] = \
file.getFullName()
rpms = []
for line in gen.generate('packages'):
if line.find('%package') == -1 and line.find('%end') == -1:
rpms.append(line)
for rpm in rpms:
if rpm in rpm_list.keys():
script.append('yum install -y %s' % rpm)
script.append(rpm_force_template % rpm_list[rpm])
cur_proc = False
for line in gen.generate('post'):
if not line.startswith('%post') and not line.startswith('%end'):
script.append(line)
else:
if cur_proc == True:
script.append('__POSTEOF__')
script.append('%s %s' % (interpreter, t_name))
cur_proc = False
try:
i = line.split().index('--interpreter')
except ValueError:
continue
interpreter = line.split()[i + 1]
t_name = tempfile.mktemp()
cur_proc = True
script.append('cat > %s << "__POSTEOF__"' % t_name)
script.append('\n# Boot scripts\n')
for line in gen.generate('boot'):
#
# skip lines that start with '%post' or '%end'
#
if line[0:5] == '%post' or line[0:4] == '%end':
continue
script.append(line)
if dryrun:
self.addText(string.join(script, '\n'))
else:
os.system(string.join(script, '\n'))
def run(self, hosts):
pathSource = os.path.join(os.sep, 'export', 'stack', 'salt')
self.pathCompiled = os.path.join(pathSource, 'compiled')
attrs = {}
for host in hosts['hosts']:
attrs[host] = {}
host_path = os.path.join(self.pathCompiled, host)
if os.path.exists(host_path):
shutil.rmtree(host_path)
for row in self.owner.call('list.host.attr', hosts['hosts']):
attrs[row['host']][row['attr']] = row['value']
for host in hosts['hosts']:
host_attrs = attrs[host]
paths = [os.path.join(pathSource, 'default')]
if 'environment' in host_attrs:
paths.append(
os.path.join(pathSource, host_attrs['environment']))
for path in paths:
tree = stack.file.Tree(path)
for dir in tree.getDirs():
for file in tree.getFiles(dir):
if not os.path.splitext(file.getName())[1] == '.xml':
continue
parser = make_parser()
handler = FileHandler(host_attrs)
parser.setContentHandler(handler)
parser.feed(handler.getXMLHeader())
parser.feed('<salt>')
fin = open(file.getFullName(), 'r')
# Parse as XML-ish to pickup the
# entities that correspond to host
# attributes. But we assume the <file
# ...> and </file> tags start at col 0
# on a line by themselves. Anything
# between the file tags is first
# sanitized for '<' and '&'
# (non-entity refs only). This keep
# the file in XML-ish mode, and
# prevents the need for using CDATA.
# Node.xml files do not behave this
# we, so while a node.xml <file>
# section can move into this framework
# the reverse is not true (yet).
for line in fin.readlines():
if line.find('<file') == 0 or \
line.find('</file>') == 0:
s = line
else:
s = ''
for i in range(0, len(line)):
if line[i] == '<':
s += '<'
elif line[i] == '&':
if not re.match(
'&[A-Za-z_.][A-Za-z0-9_.]*;',
line[i:]):
s += '&'
else:
s += line[i]
else:
s += line[i]
try:
parser.feed(s)
except:
print('Parsing Error file %s line %s' %
(file.getFullName(), line))
parser.feed('</salt>')
fin.close()
for parsed in handler.getFiles():
if stack.cond.EvalCondExpr(parsed.get('cond'),
host_attrs):
self.writeSalt(host, parsed)
def run(self, params, args):
(dryrun, ) = self.fillParams([ ('dryrun', 'true')])
dryrun = self.str2bool(dryrun)
script = []
script.append('#!/bin/sh')
rolls = []
for roll in args:
rolls.append(roll)
if sys.stdin.isatty():
xml = self.command('list.host.xml', [ 'localhost',
'pallet=%s' % string.join(rolls, ',') ])
else:
xml = sys.stdin.read()
reader = Sax2.Reader()
gen = getattr(stack.gen,'Generator_%s' % self.os)()
gen.parse(xml)
distPath = os.path.join(self.command('report.distribution')[:-1],
'default')
tree = stack.file.Tree(distPath)
rpm_list = {}
for file in tree.getFiles(os.path.join(self.arch,
'RedHat', 'RPMS')):
if isinstance(file, stack.file.RPMFile):
rpm_list[file.getBaseName()] = \
file.getFullName()
rpm_list["%s.%s" % (file.getBaseName(), \
file.getPackageArch())] = \
file.getFullName()
rpms = []
for line in gen.generate('packages'):
if line.find('%package') == -1 and line.find('%end') == -1:
rpms.append(line)
for rpm in rpms:
if rpm in rpm_list.keys():
script.append('yum install -y %s' %
rpm)
script.append(rpm_force_template %
rpm_list[rpm])
cur_proc = False
for line in gen.generate('post'):
if not line.startswith('%post') and not line.startswith('%end'):
script.append(line)
else:
if cur_proc == True:
script.append('__POSTEOF__')
script.append('%s %s' % (interpreter,
t_name))
cur_proc = False
try:
i = line.split().index('--interpreter')
except ValueError:
continue
interpreter = line.split()[i+1]
t_name = tempfile.mktemp()
cur_proc = True
script.append('cat > %s << "__POSTEOF__"' %
t_name)
script.append('\n# Boot scripts\n')
for line in gen.generate('boot'):
#
# skip lines that start with '%post' or '%end'
#
if line[0:5] == '%post' or line[0:4] == '%end':
continue
script.append(line)
if dryrun:
self.addText(string.join(script, '\n'))
else:
os.system(string.join(script, '\n'))
def run(self, args):
(clean, prefix, treeinfo) = args
name = None
vers = None
arch = None
file = open(treeinfo, 'r')
for line in file.readlines():
a = line.split('=')
if len(a) != 2:
continue
key = a[0].strip()
value = a[1].strip()
if key == 'family':
if value == 'Red Hat Enterprise Linux':
name = 'RHEL'
elif value == 'CentOS':
name = 'CentOS'
elif key == 'version':
vers = value
elif key == 'arch':
arch = value
file.close()
if not name:
name = "BaseOS"
if not vers:
vers = stack.version
if not arch:
arch = 'x86_64'
OS = 'redhat'
roll_dir = os.path.join(prefix, name, vers, OS, arch)
destdir = os.path.join(roll_dir, 'RPMS')
if clean and os.path.exists(roll_dir):
print 'Cleaning %s version %s ' % (name, vers),
print 'for %s from pallets directory' % arch
os.system('/bin/rm -rf %s' % roll_dir)
os.makedirs(roll_dir)
print 'Copying "%s" (%s,%s) pallet ...' % (name, vers, arch)
if not os.path.exists(destdir):
os.makedirs(destdir)
cdtree = stack.file.Tree(self.owner.mountPoint)
for dir in cdtree.getDirs():
for file in cdtree.getFiles(dir):
if not file.getName().endswith('.rpm'):
continue
if file.getPackageArch() != 'src' and \
file.getBaseName() != 'comps' and \
file.getName() != 'comps.rpm' \
and not (arch == 'i386' and \
re.match('^kudzu.*', file.getBaseName())):
os.system('cp -p %s %s' % (
file.getFullName(),
destdir))
#
# lay down a minimal roll XML config file
#
f = open('%s' % os.path.join(roll_dir, 'roll-%s.xml' % name),
'w')
f.write('<roll name="%s" interface="6.0.2">\n' % name)
f.write('<info version="%s" release="%s" arch="%s" os="%s"/>\n' % (vers, stack.release, arch, OS))
f.write('<iso maxsize="0" bootable="0"/>\n')
f.write('</roll>\n')
f.close()
return (name, vers, arch)
def run(self, args):
import stack
(clean, prefix, treeinfo) = args
name = None
vers = None
arch = None
file = open(treeinfo, 'r')
for line in file.readlines():
a = line.split('=')
if len(a) != 2:
continue
key = a[0].strip()
value = a[1].strip()
if key == 'family':
if value == 'Red Hat Enterprise Linux':
name = 'RHEL'
elif value == 'CentOS':
name = 'CentOS'
elif key == 'version':
vers = value
elif key == 'arch':
arch = value
file.close()
if not name:
name = "BaseOS"
if not vers:
vers = stack.version
if not arch:
arch = 'x86_64'
OS = 'redhat'
roll_dir = os.path.join(prefix, name, vers, OS, arch)
destdir = os.path.join(roll_dir, 'RPMS')
print 'stack.release (%s)' % stack.release
if stack.release == '7.x':
liveosdir = os.path.join(roll_dir, 'LiveOS')
if clean and os.path.exists(roll_dir):
print 'Cleaning %s version %s ' % (name, vers),
print 'for %s from pallets directory' % arch
os.system('/bin/rm -rf %s' % roll_dir)
os.makedirs(roll_dir)
print 'Copying "%s" (%s,%s) pallet ...' % (name, vers, arch)
if not os.path.exists(destdir):
os.makedirs(destdir)
if stack.release == '7.x' and not os.path.exists(liveosdir):
os.makedirs(liveosdir)
cdtree = stack.file.Tree(self.owner.mountPoint)
for dir in cdtree.getDirs():
for file in cdtree.getFiles(dir):
if stack.release == '7.x' and dir == 'LiveOS' \
and file.getName().endswith('.img'):
os.system('cp -p %s %s' % (
file.getFullName(),
liveosdir))
else:
if not file.getName().endswith('.rpm'):
continue
if file.getPackageArch() != 'src' and \
file.getBaseName() != 'comps' and \
file.getName() != 'comps.rpm' \
and not (arch == 'i386' and \
re.match('^kudzu.*', file.getBaseName())):
os.system('cp -p %s %s' % (
file.getFullName(),
destdir))
#
# lay down a minimal roll XML config file
#
f = open('%s' % os.path.join(roll_dir, 'roll-%s.xml' % name),
'w')
f.write('<roll name="%s" interface="6.0.2">\n' % name)
f.write('<info version="%s" release="%s" arch="%s" os="%s"/>\n' % (vers, stack.release, arch, OS))
f.write('<iso maxsize="0" bootable="0"/>\n')
f.write('</roll>\n')
f.close()
return (name, vers, arch)
