def bundleAMI(chan):
options, rchan = chan.receive()
try:
cmd = ['ec2-bundle-image',
'-c ${general.cert}',
'-k ${general.key}',
'-u ${general.user}',
'--kernel ${general.kernel}',
'-i ${general.image}',
'-d ${general.dest}',
'-p ${general.image}',
'-r ${general.arch}']
if options('general.ec2cert'):
cmd.append('--ec2cert ${general.ec2cert}')
runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug'))
cmd = ['ec2-upload-bundle', '-b ${general.image}', '-m ${general.dest}/${general.image}.manifest.xml', '-a ${general.access_key}', '-s ${general.secret_access_key}']
runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug'))
cmd = ['ec2-register', '${general.image}/${general.image}.manifest.xml', '-K ${general.key}', '-C ${general.cert}']
outp = []
runSingleProgramEx(config.replaceStr(' '.join(cmd), options), stdoutf=outp.append, stderrf=sys.stderr.write, log=True)
ami = ''.join(outp).split()[-1]
##
# Make the AMI public
runSingleProgramEx('ec2-modify-image-attribute %s --launch-permission -a all' % ami, stdoutf=sys.stdout.write, stderrf=sys.stderr.write, log=True)
rchan.send(ami)
except Exception, err:
rchan.sendError(err)
def fileOwner(fname, owner, group=None):
fname = replaceStr(fname, conf)
owner = replaceStr(owner, conf)
who = owner
if group:
group = replaceStr(group, conf)
who += ':' + group
runSystemEx('chown %s %s' % (who, fname))
def fileOwner(fname, owner, group=None):
fname = replaceStr(fname, conf)
owner = replaceStr(owner, conf)
who = owner
if group:
group = replaceStr(group, conf)
who += ':' + group
runSystemEx('chown %s %s' % (who, fname))
def dirOwner(dirname, owner, group=None, ignoreError=False):
"""Set owners of a directory, recursively. use fileOwner if you do not want recursive"""
dirname = replaceStr(dirname, conf)
owner = replaceStr(owner, conf)
who = owner
if group:
group = replaceStr(group, conf)
who += ':' + group
run('chown -R %s %s' % (who, dirname), ignoreError=ignoreError)
def dirOwner(dirname, owner, group=None, ignoreError=False):
"""Set owners of a directory, recursively. use fileOwner if you do not want recursive"""
dirname = replaceStr(dirname, conf)
owner = replaceStr(owner, conf)
who = owner
if group:
group = replaceStr(group, conf)
who += ':' + group
run('chown -R %s %s' % (who, dirname), ignoreError=ignoreError)
def c(conf):
val = x
for f in funcs:
val = f(val)
try:
val = replaceStr(val, conf)
except TypeError:
pass
val = applyIfCallable(val, conf)
try:
return replaceStr(val, conf)
except TypeError:
return val
def c(conf):
val = x
for f in funcs:
val = f(val)
try:
val = replaceStr(val, conf)
except TypeError:
pass
val = applyIfCallable(val, conf)
try:
return replaceStr(val, conf)
except TypeError:
return val
def run(cmd, ignoreError=False):
"""This runs a command, be sure that the command backgrounds, add & if you need to"""
try:
runSystemEx(replaceStr(cmd, conf))
except ProgramRunError:
if not ignoreError:
raise
def main(options, _args):
runSystemEx('svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')),
log=True)
runSystemEx('svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')),
log=True)
runSystemEx('scp %s:/export/%s .' % (options('general.remote_name'), options('general.image')), log=True)
runSystemEx('cp %s /usr/local/projects/clovr/images' % options('general.image'), log=True)
runSystemEx('cp %s VMware_conversion/shared/convert_img.img' % options('general.image'), log=True)
convertChannel = threads.runThreadWithChannel(convertImage).channel.sendWithChannel(options)
waitForPasswordChange()
bundleChannel = threads.runThreadWithChannel(bundleAMI).channel.sendWithChannel(options)
try:
convertChannel.receive()
vmWareDir = 'clovr-vmware.%s' % options('general.version')
runSystemEx('mkdir -p ' + vmWareDir, log=True)
runSystemEx('mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join(vmWareDir, 'clovr.9-04.x86-64.%s.vmdk' % options('general.version')))
runSystemEx('mkdir -p %s %s' % (os.path.join(vmWareDir, 'keys'),
os.path.join(vmWareDir, 'user_data')), log=True)
runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir, log=True)
fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w')
clovrConf = config.configFromMap(dict(version=options('general.version')))
for line in open('/usr/local/projects/clovr/start_clovr.vmx'):
fout.write(config.replaceStr(line, clovrConf))
except Exception, err:
errorPrint('Converting image failed. Error message:')
errorPrint(str(err))
def run(cmd, ignoreError=False):
"""This runs a command, be sure that the command backgrounds, add & if you need to"""
try:
runSystemEx(replaceStr(cmd, conf))
except ProgramRunError:
if not ignoreError:
raise
def runPipelineWithConfig(taskName, name, pipeline, conf, queue):
"""
This is for internal use only
"""
templateDir = os.path.join(conf("dirs.clovr_pipelines_template_dir"), pipeline.TEMPLATE_NAME)
templateConfig = os.path.join(templateDir, "pipeline_tmpl.config")
templateLayout = os.path.join(templateDir, "pipeline.layout")
foutName = os.path.join("/tmp", str(time.time()))
fout = open(foutName, "w")
for line in handleIncludes(open(templateConfig)):
fout.write(config.replaceStr(line, conf) + "\n")
fout.close()
cmd = ["run_pipeline.pl", "--config=" + foutName, "--templatelayout=" + templateLayout, "--taskname=" + taskName]
if queue:
cmd.append("--queue=" + queue)
cmd = " ".join(cmd)
res = []
exitCode = runSingleProgram(cmd, res.append, None)
##
# If we got a weird exit code or more than one line was print or nothing was printed
# then something bad happened
if exitCode != 0 or len(res) > 1 or not res:
raise ProgramRunError(cmd, exitCode)
##
# This should be the pipeline ID
return createPipeline(taskName, name, res[0].strip(), pipeline, conf)
def run(state, pipeline):
if not os.path.exists(state.conf('config.pipeline_configs')):
os.mkdir(state.conf('config.pipeline_configs'))
tmpConfigName = os.path.join(state.conf('config.pipeline_configs'), str(time.time()) + '.config')
pipeline.config = config.configFromMap({'CONFIG_FILE': tmpConfigName},
base=pipeline.config)
fout = open(tmpConfigName, 'w')
# We want to produce an ini like file with [section]'s
sections = {}
for k in pipeline.config.keys():
sections.setdefault('.'.join(k.split('.')[:-1]), []).append(k)
for s, ks in sections.iteritems():
if s not in ['', 'env']:
fout.write('[' + s + ']\n')
for k in ks:
shortK = k.split('.')[-1]
fout.write('%s=%s\n' % (shortK, str(pipeline.config(k))))
fout.close()
templateDir = os.path.join(state.machineconf('dirs.clovr_pipelines_template_dir'),
pipeline.protocol)
templateConfig = os.path.join(templateDir, 'pipeline_tmpl.config')
templateLayout = os.path.join(templateDir, 'pipeline.layout')
tmpPipelineConfig = os.path.join(state.conf('config.pipeline_configs'), str(time.time()) + '.pipeline.config')
fout = open(tmpPipelineConfig, 'w')
for line in handleIncludes(open(templateConfig)):
fout.write(config.replaceStr(line, pipeline.config) + '\n')
fout.close()
cmd = ['run_pipeline.pl',
'--config=' + tmpPipelineConfig,
'--templatelayout=' + templateLayout,
'--taskname=' + pipeline.taskName]
if pipeline.queue:
cmd.append('--queue=' + pipeline.queue)
stdout = StringIO.StringIO()
stderr = StringIO.StringIO()
yield commands.runProcess(cmd,
stdoutf=stdout.write,
stderrf=stderr.write)
pipelineId = stdout.getvalue().strip()
if not pipelineId:
raise commands.ProgramRunError(cmd, stderr.getvalue())
defer.returnValue(pipeline.update(pipelineId=pipelineId))
def bundleAMI(chan):
options, rchan = chan.receive()
try:
cmd = [
'ec2-bundle-image', '-c ${general.cert}', '-k ${general.key}',
'-u ${general.user}', '--kernel ${general.kernel}',
'-i ${general.image}', '-d ${general.dest}', '-p ${general.image}',
'-r ${general.arch}'
]
if options('general.ec2cert'):
cmd.append('--ec2cert ${general.ec2cert}')
runSystemEx(config.replaceStr(' '.join(cmd), options),
log=options('general.debug'))
cmd = [
'ec2-upload-bundle', '-b ${general.image}',
'-m ${general.dest}/${general.image}.manifest.xml',
'-a ${general.access_key}', '-s ${general.secret_access_key}'
]
runSystemEx(config.replaceStr(' '.join(cmd), options),
log=options('general.debug'))
cmd = [
'ec2-register', '${general.image}/${general.image}.manifest.xml',
'-K ${general.key}', '-C ${general.cert}'
]
outp = []
runSingleProgramEx(config.replaceStr(' '.join(cmd), options),
stdoutf=outp.append,
stderrf=sys.stderr.write,
log=True)
ami = ''.join(outp).split()[-1]
##
# Make the AMI public
runSingleProgramEx(
'ec2-modify-image-attribute %s --launch-permission -a all' % ami,
stdoutf=sys.stdout.write,
stderrf=sys.stderr.write,
log=True)
rchan.send(ami)
except Exception, err:
rchan.sendError(err)
def dirExists(dirname):
"""
Ensure a directory exists, create it if not
Use fileExists if you want to check for existence but not create
"""
dirname = replaceStr(dirname, conf)
if not os.path.exists(dirname):
try:
runSystemEx('mkdir -p ' + dirname)
except:
raise PolicyError('Could not create directory: ' + dirname)
def executeTemplate(fname):
"""
Executes a template, at this point this is simple variable substitution in a file
but it could grow to be more complicated.
This takes a file that ends in .tmpl and produces a file without the .tmpl with the template
executed
"""
fname = replaceStr(fname, conf)
if not fname.endswith('.tmpl'):
raise PolicyError('%s does not end in .tmpl' % fname)
fout = open(fname[:-5], 'w')
fin = open(fname)
for line in fin:
fout.write(replaceStr(line, conf))
fout.close()
fin.close()
def dirExists(dirname):
"""
Ensure a directory exists, create it if not
Use fileExists if you want to check for existence but not create
"""
dirname = replaceStr(dirname, conf)
if not os.path.exists(dirname):
try:
runSystemEx('mkdir -p ' + dirname)
except:
raise PolicyError('Could not create directory: ' + dirname)
def executeTemplate(fname):
"""
Executes a template, at this point this is simple variable substitution in a file
but it could grow to be more complicated.
This takes a file that ends in .tmpl and produces a file without the .tmpl with the template
executed
"""
fname = replaceStr(fname, conf)
if not fname.endswith('.tmpl'):
raise PolicyError('%s does not end in .tmpl' % fname)
fout = open(fname[:-5], 'w')
fin = open(fname)
for line in fin:
fout.write(replaceStr(line, conf))
fout.close()
fin.close()
def _(conf):
files = open(replaceStr(tagFname, conf)).readlines()
prefix = files.pop(0)
for f in files:
prefix = commonPrefix(prefix, f)
if prefix[-1] == '.':
prefix = prefix[:-1]
elif prefix.endswith('.n') or prefix.endswith('.p'):
prefix = prefix[:-2]
return prefix
def main(options, _args):
cmd = ['ec2-bundle-image',
'-c ${general.cert}',
'-k ${general.key}',
'-u ${general.user}',
'--kernel ${general.kernel}',
'-i ${general.image}',
'-d ${general.dest}',
'-p ${general.image}',
'-r ${general.arch}']
if options('general.ec2cert'):
cmd.append('--ec2cert ${general.ec2cert}')
runSystemEx(replaceStr(' '.join(cmd), options), log=options('general.debug'))
cmd = ['ec2-upload-bundle', '-b ${general.image}', '-m ${general.dest}/${general.image}.manifest.xml', '-a ${general.access_key}', '-s ${general.secret_access_key}']
runSystemEx(replaceStr(' '.join(cmd), options), log=options('general.debug'))
cmd = ['ec2-register', '${general.image}/${general.image}.manifest.xml', '-K ${general.key}', '-C ${general.cert}']
##
# We want to output the AMI regardless
runSystemEx(replaceStr(' '.join(cmd), options), log=True)
def main(options, _args):
runSystemEx(
'svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"'
% (options('general.version'), options('general.version')),
log=True)
runSystemEx(
'svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"'
% (options('general.version'), options('general.version')),
log=True)
runSystemEx('scp %s:/export/%s .' %
(options('general.remote_name'), options('general.image')),
log=True)
runSystemEx('cp %s /usr/local/projects/clovr/images' %
options('general.image'),
log=True)
runSystemEx('cp %s VMware_conversion/shared/convert_img.img' %
options('general.image'),
log=True)
convertChannel = threads.runThreadWithChannel(
convertImage).channel.sendWithChannel(options)
waitForPasswordChange()
bundleChannel = threads.runThreadWithChannel(
bundleAMI).channel.sendWithChannel(options)
try:
convertChannel.receive()
vmWareDir = 'clovr-vmware.%s' % options('general.version')
runSystemEx('mkdir -p ' + vmWareDir, log=True)
runSystemEx(
'mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join(
vmWareDir,
'clovr.9-04.x86-64.%s.vmdk' % options('general.version')))
runSystemEx('mkdir -p %s %s' % (os.path.join(
vmWareDir, 'keys'), os.path.join(vmWareDir, 'user_data')),
log=True)
runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir,
log=True)
fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w')
clovrConf = config.configFromMap(
dict(version=options('general.version')))
for line in open('/usr/local/projects/clovr/start_clovr.vmx'):
fout.write(config.replaceStr(line, clovrConf))
except Exception, err:
errorPrint('Converting image failed. Error message:')
errorPrint(str(err))
def applyOption(val, option, conf):
try:
#
# We want to apply any replacements on the options
# The question is if baseConf is really the config file
# we should be applying these from...
v = option[4](val)
#
# Now v might still be a function at this point because we want to give
# them the ability to access baseConf is they need it to do more replacements
v = applyIfCallable(v, conf)
try:
return replaceStr(v, conf)
except TypeError:
return v
except MissingOptionError, err:
raise CLIError(option[0], err)
def applyOption(val, option, conf):
try:
#
# We want to apply any replacements on the options
# The question is if baseConf is really the config file
# we should be applying these from...
v = option[4](val)
#
# Now v might still be a function at this point because we want to give
# them the ability to access baseConf is they need it to do more replacements
v = applyIfCallable(v, conf)
try:
return replaceStr(v, conf)
except TypeError:
return v
except MissingOptionError, err:
raise CLIError(option[0], err)
def runPipelineWithConfig(taskName, name, pipeline, conf, queue):
"""
This is for internal use only
"""
templateDir = os.path.join(conf('dirs.clovr_pipelines_template_dir'),
pipeline.TEMPLATE_NAME)
templateConfig = os.path.join(templateDir, 'pipeline_tmpl.config')
templateLayout = os.path.join(templateDir, 'pipeline.layout')
foutName = os.path.join('/tmp', str(time.time()))
fout = open(foutName, 'w')
for line in handleIncludes(open(templateConfig)):
fout.write(config.replaceStr(line, conf) + '\n')
fout.close()
cmd = [
'run_pipeline.pl', '--config=' + foutName,
'--templatelayout=' + templateLayout, '--taskname=' + taskName
]
if queue:
cmd.append('--queue=' + queue)
cmd = ' '.join(cmd)
res = []
exitCode = runSingleProgram(cmd, res.append, None)
##
# If we got a weird exit code or more than one line was print or nothing was printed
# then something bad happened
if exitCode != 0 or len(res) > 1 or not res:
raise ProgramRunError(cmd, exitCode)
##
# This should be the pipeline ID
return createPipeline(taskName, name, res[0].strip(), pipeline, conf)
def buildConfigN(options, args=None, usage=None, baseConf=None, putInGeneral=True):
"""
This builds a config from options. Options is a list of tuples that looks like:
(name, short, long, help, func, [bool])
Where
name - Name of the option, this is what it will become in the config file
short - Short option - needs to start with -
long - Long option - needs to start with --
help - Help to be given to a user in --help output
func - Function to be applied to the value
bool - This is not required, set to True if the option is simply a boolean, all other datatypes can be verified via 'func'
This will implicitly check if a 'conf' option exists, and if so load te conf file as a base for these config options.
All options are put into the 'general' section.
This returns a tuple
(conf, args)
where args is whatever is left over from parsing
This also implicitly loads the current environment into the env section
If, when evaluated, 'func' returns a function, it is called with the baseConf. This is to allow more complex replacements to
happen.
"""
def _iterBool(v):
"""
Adds the non erquired bool field with a default of STRING if
it is not present
"""
for l in v:
if len(l) == 6:
yield l
else:
yield tuple(list(l) + [STRING])
parser = optparse.OptionParser(usage=usage)
##
# keep track of the function to apply to conf
confFunc = None
##
# The order of the options below
# var name, short option, long option, help message, function to apply, binary option
for n, s, l, h, f, b in _iterBool(options):
##
# We could have a function we want to apply to the conf variable. We want to store it
# so when we use it in the next block we don't have to loop over options looking for it again
# This is a minor optimization and probably not even necessary...
if n == 'conf':
confFunc = f
if b == BINARY:
parser.add_option(s, l, dest=n, help=h, action='store_true')
elif b == LIST:
parser.add_option(s, l, dest=n, help=h, action='append')
elif b == COUNT:
parser.add_option(s, l, dest=n, help=h, action='count')
elif b == STRING:
parser.add_option(s, l, dest=n, help=h)
else:
raise Exception('Unknown option type: ' + repr(b))
ops, args = parser.parse_args(args=args)
if baseConf is None:
baseConf = configFromEnv()
if hasattr(ops, 'conf'):
baseConf = configFromStream(open(replaceStr(confFunc(ops.conf), baseConf)), baseConf)
vals = {}
##
# The order of the options below
# var name, short option, long option, help message, function to apply, binary option
for o in _iterBool(options):
n, _s, l, _h, f, _b = o
try:
vals[n] = applyOption(getattr(ops, n), o, baseConf)
except Exception, err:
raise CLIError(l, err)
def pkgFileExists(pkgname, fname):
"""Ensures a file in the package exists"""
fname = replaceStr(fname, conf)
fileExists(os.path.join(conf('stow.package_dir'), pkgname, fname))
def runSystemExC(conf, cmd):
cmd = replaceStr(cmd, conf)
runSystemEx(cmd)
def runSystemExC(conf, cmd):
cmd = replaceStr(cmd, conf)
runSystemEx(cmd)
def runSingleProgramEx(conf, cmd, stdoutf, stderrf):
cmd = replaceStr(cmd, conf)
exitCode = runSingleProgram(cmd, stdoutf, stderrf)
if exitCode != 0:
raise ProgramRunError(cmd, exitCode)
def dirPermissions(dirname, perms, ignoreError=False):
dirname = replaceStr(dirname, conf)
perms = replaceStr(perms, conf)
run('chmod -R %s %s' % (perms, dirname), ignoreError=ignoreError)
def fileExists(fname):
fname = replaceStr(fname, conf)
if not os.path.exists(fname):
raise PolicyError('File does not exist: ' + fname)
def dirPermissions(dirname, perms, ignoreError=False):
dirname = replaceStr(dirname, conf)
perms = replaceStr(perms, conf)
run('chmod -R %s %s' % (perms, dirname), ignoreError=ignoreError)
def fileExists(fname):
fname = replaceStr(fname, conf)
if not os.path.exists(fname):
raise PolicyError('File does not exist: ' + fname)
def buildConfigN(options,
args=None,
usage=None,
baseConf=None,
putInGeneral=True):
"""
This builds a config from options. Options is a list of tuples that looks like:
(name, short, long, help, func, [bool])
Where
name - Name of the option, this is what it will become in the config file
short - Short option - needs to start with -
long - Long option - needs to start with --
help - Help to be given to a user in --help output
func - Function to be applied to the value
bool - This is not required, set to True if the option is simply a boolean, all other datatypes can be verified via 'func'
This will implicitly check if a 'conf' option exists, and if so load te conf file as a base for these config options.
All options are put into the 'general' section.
This returns a tuple
(conf, args)
where args is whatever is left over from parsing
This also implicitly loads the current environment into the env section
If, when evaluated, 'func' returns a function, it is called with the baseConf. This is to allow more complex replacements to
happen.
"""
def _iterBool(v):
"""
Adds the non erquired bool field with a default of STRING if
it is not present
"""
for l in v:
if len(l) == 6:
yield l
else:
yield tuple(list(l) + [STRING])
parser = optparse.OptionParser(usage=usage)
##
# keep track of the function to apply to conf
confFunc = None
##
# The order of the options below
# var name, short option, long option, help message, function to apply, binary option
for n, s, l, h, f, b in _iterBool(options):
##
# We could have a function we want to apply to the conf variable. We want to store it
# so when we use it in the next block we don't have to loop over options looking for it again
# This is a minor optimization and probably not even necessary...
if n == 'conf':
confFunc = f
if b == BINARY:
parser.add_option(s, l, dest=n, help=h, action='store_true')
elif b == LIST:
parser.add_option(s, l, dest=n, help=h, action='append')
elif b == COUNT:
parser.add_option(s, l, dest=n, help=h, action='count')
elif b == STRING:
parser.add_option(s, l, dest=n, help=h)
else:
raise Exception('Unknown option type: ' + repr(b))
ops, args = parser.parse_args(args=args)
if baseConf is None:
baseConf = configFromEnv()
if hasattr(ops, 'conf'):
baseConf = configFromStream(
open(replaceStr(confFunc(ops.conf), baseConf)), baseConf)
vals = {}
##
# The order of the options below
# var name, short option, long option, help message, function to apply, binary option
for o in _iterBool(options):
n, _s, l, _h, f, _b = o
try:
vals[n] = applyOption(getattr(ops, n), o, baseConf)
except Exception, err:
raise CLIError(l, err)
def pkgFileExists(pkgname, fname):
"""Ensures a file in the package exists"""
fname = replaceStr(fname, conf)
fileExists(os.path.join(conf('stow.package_dir'), pkgname, fname))
def runSingleProgramEx(conf, cmd, stdoutf, stderrf):
cmd = replaceStr(cmd, conf)
exitCode = runSingleProgram(cmd, stdoutf, stderrf)
if exitCode != 0:
raise ProgramRunError(cmd, exitCode)
