def bundleAMI(chan): options, rchan = chan.receive() try: cmd = ['ec2-bundle-image', '-c ${general.cert}', '-k ${general.key}', '-u ${general.user}', '--kernel ${general.kernel}', '-i ${general.image}', '-d ${general.dest}', '-p ${general.image}', '-r ${general.arch}'] if options('general.ec2cert'): cmd.append('--ec2cert ${general.ec2cert}') runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = ['ec2-upload-bundle', '-b ${general.image}', '-m ${general.dest}/${general.image}.manifest.xml', '-a ${general.access_key}', '-s ${general.secret_access_key}'] runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = ['ec2-register', '${general.image}/${general.image}.manifest.xml', '-K ${general.key}', '-C ${general.cert}'] outp = [] runSingleProgramEx(config.replaceStr(' '.join(cmd), options), stdoutf=outp.append, stderrf=sys.stderr.write, log=True) ami = ''.join(outp).split()[-1] ## # Make the AMI public runSingleProgramEx('ec2-modify-image-attribute %s --launch-permission -a all' % ami, stdoutf=sys.stdout.write, stderrf=sys.stderr.write, log=True) rchan.send(ami) except Exception, err: rchan.sendError(err)
def fileOwner(fname, owner, group=None): fname = replaceStr(fname, conf) owner = replaceStr(owner, conf) who = owner if group: group = replaceStr(group, conf) who += ':' + group runSystemEx('chown %s %s' % (who, fname))
def dirOwner(dirname, owner, group=None, ignoreError=False): """Set owners of a directory, recursively. use fileOwner if you do not want recursive""" dirname = replaceStr(dirname, conf) owner = replaceStr(owner, conf) who = owner if group: group = replaceStr(group, conf) who += ':' + group run('chown -R %s %s' % (who, dirname), ignoreError=ignoreError)
def c(conf): val = x for f in funcs: val = f(val) try: val = replaceStr(val, conf) except TypeError: pass val = applyIfCallable(val, conf) try: return replaceStr(val, conf) except TypeError: return val
def run(cmd, ignoreError=False): """This runs a command, be sure that the command backgrounds, add & if you need to""" try: runSystemEx(replaceStr(cmd, conf)) except ProgramRunError: if not ignoreError: raise
def main(options, _args): runSystemEx('svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx('svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx('scp %s:/export/%s .' % (options('general.remote_name'), options('general.image')), log=True) runSystemEx('cp %s /usr/local/projects/clovr/images' % options('general.image'), log=True) runSystemEx('cp %s VMware_conversion/shared/convert_img.img' % options('general.image'), log=True) convertChannel = threads.runThreadWithChannel(convertImage).channel.sendWithChannel(options) waitForPasswordChange() bundleChannel = threads.runThreadWithChannel(bundleAMI).channel.sendWithChannel(options) try: convertChannel.receive() vmWareDir = 'clovr-vmware.%s' % options('general.version') runSystemEx('mkdir -p ' + vmWareDir, log=True) runSystemEx('mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join(vmWareDir, 'clovr.9-04.x86-64.%s.vmdk' % options('general.version'))) runSystemEx('mkdir -p %s %s' % (os.path.join(vmWareDir, 'keys'), os.path.join(vmWareDir, 'user_data')), log=True) runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir, log=True) fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w') clovrConf = config.configFromMap(dict(version=options('general.version'))) for line in open('/usr/local/projects/clovr/start_clovr.vmx'): fout.write(config.replaceStr(line, clovrConf)) except Exception, err: errorPrint('Converting image failed. Error message:') errorPrint(str(err))
def runPipelineWithConfig(taskName, name, pipeline, conf, queue): """ This is for internal use only """ templateDir = os.path.join(conf("dirs.clovr_pipelines_template_dir"), pipeline.TEMPLATE_NAME) templateConfig = os.path.join(templateDir, "pipeline_tmpl.config") templateLayout = os.path.join(templateDir, "pipeline.layout") foutName = os.path.join("/tmp", str(time.time())) fout = open(foutName, "w") for line in handleIncludes(open(templateConfig)): fout.write(config.replaceStr(line, conf) + "\n") fout.close() cmd = ["run_pipeline.pl", "--config=" + foutName, "--templatelayout=" + templateLayout, "--taskname=" + taskName] if queue: cmd.append("--queue=" + queue) cmd = " ".join(cmd) res = [] exitCode = runSingleProgram(cmd, res.append, None) ## # If we got a weird exit code or more than one line was print or nothing was printed # then something bad happened if exitCode != 0 or len(res) > 1 or not res: raise ProgramRunError(cmd, exitCode) ## # This should be the pipeline ID return createPipeline(taskName, name, res[0].strip(), pipeline, conf)
def run(state, pipeline): if not os.path.exists(state.conf('config.pipeline_configs')): os.mkdir(state.conf('config.pipeline_configs')) tmpConfigName = os.path.join(state.conf('config.pipeline_configs'), str(time.time()) + '.config') pipeline.config = config.configFromMap({'CONFIG_FILE': tmpConfigName}, base=pipeline.config) fout = open(tmpConfigName, 'w') # We want to produce an ini like file with [section]'s sections = {} for k in pipeline.config.keys(): sections.setdefault('.'.join(k.split('.')[:-1]), []).append(k) for s, ks in sections.iteritems(): if s not in ['', 'env']: fout.write('[' + s + ']\n') for k in ks: shortK = k.split('.')[-1] fout.write('%s=%s\n' % (shortK, str(pipeline.config(k)))) fout.close() templateDir = os.path.join(state.machineconf('dirs.clovr_pipelines_template_dir'), pipeline.protocol) templateConfig = os.path.join(templateDir, 'pipeline_tmpl.config') templateLayout = os.path.join(templateDir, 'pipeline.layout') tmpPipelineConfig = os.path.join(state.conf('config.pipeline_configs'), str(time.time()) + '.pipeline.config') fout = open(tmpPipelineConfig, 'w') for line in handleIncludes(open(templateConfig)): fout.write(config.replaceStr(line, pipeline.config) + '\n') fout.close() cmd = ['run_pipeline.pl', '--config=' + tmpPipelineConfig, '--templatelayout=' + templateLayout, '--taskname=' + pipeline.taskName] if pipeline.queue: cmd.append('--queue=' + pipeline.queue) stdout = StringIO.StringIO() stderr = StringIO.StringIO() yield commands.runProcess(cmd, stdoutf=stdout.write, stderrf=stderr.write) pipelineId = stdout.getvalue().strip() if not pipelineId: raise commands.ProgramRunError(cmd, stderr.getvalue()) defer.returnValue(pipeline.update(pipelineId=pipelineId))
def bundleAMI(chan): options, rchan = chan.receive() try: cmd = [ 'ec2-bundle-image', '-c ${general.cert}', '-k ${general.key}', '-u ${general.user}', '--kernel ${general.kernel}', '-i ${general.image}', '-d ${general.dest}', '-p ${general.image}', '-r ${general.arch}' ] if options('general.ec2cert'): cmd.append('--ec2cert ${general.ec2cert}') runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = [ 'ec2-upload-bundle', '-b ${general.image}', '-m ${general.dest}/${general.image}.manifest.xml', '-a ${general.access_key}', '-s ${general.secret_access_key}' ] runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = [ 'ec2-register', '${general.image}/${general.image}.manifest.xml', '-K ${general.key}', '-C ${general.cert}' ] outp = [] runSingleProgramEx(config.replaceStr(' '.join(cmd), options), stdoutf=outp.append, stderrf=sys.stderr.write, log=True) ami = ''.join(outp).split()[-1] ## # Make the AMI public runSingleProgramEx( 'ec2-modify-image-attribute %s --launch-permission -a all' % ami, stdoutf=sys.stdout.write, stderrf=sys.stderr.write, log=True) rchan.send(ami) except Exception, err: rchan.sendError(err)
def dirExists(dirname): """ Ensure a directory exists, create it if not Use fileExists if you want to check for existence but not create """ dirname = replaceStr(dirname, conf) if not os.path.exists(dirname): try: runSystemEx('mkdir -p ' + dirname) except: raise PolicyError('Could not create directory: ' + dirname)
def executeTemplate(fname): """ Executes a template, at this point this is simple variable substitution in a file but it could grow to be more complicated. This takes a file that ends in .tmpl and produces a file without the .tmpl with the template executed """ fname = replaceStr(fname, conf) if not fname.endswith('.tmpl'): raise PolicyError('%s does not end in .tmpl' % fname) fout = open(fname[:-5], 'w') fin = open(fname) for line in fin: fout.write(replaceStr(line, conf)) fout.close() fin.close()
def _(conf): files = open(replaceStr(tagFname, conf)).readlines() prefix = files.pop(0) for f in files: prefix = commonPrefix(prefix, f) if prefix[-1] == '.': prefix = prefix[:-1] elif prefix.endswith('.n') or prefix.endswith('.p'): prefix = prefix[:-2] return prefix
def main(options, _args): cmd = ['ec2-bundle-image', '-c ${general.cert}', '-k ${general.key}', '-u ${general.user}', '--kernel ${general.kernel}', '-i ${general.image}', '-d ${general.dest}', '-p ${general.image}', '-r ${general.arch}'] if options('general.ec2cert'): cmd.append('--ec2cert ${general.ec2cert}') runSystemEx(replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = ['ec2-upload-bundle', '-b ${general.image}', '-m ${general.dest}/${general.image}.manifest.xml', '-a ${general.access_key}', '-s ${general.secret_access_key}'] runSystemEx(replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = ['ec2-register', '${general.image}/${general.image}.manifest.xml', '-K ${general.key}', '-C ${general.cert}'] ## # We want to output the AMI regardless runSystemEx(replaceStr(' '.join(cmd), options), log=True)
def main(options, _args): runSystemEx( 'svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx( 'svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx('scp %s:/export/%s .' % (options('general.remote_name'), options('general.image')), log=True) runSystemEx('cp %s /usr/local/projects/clovr/images' % options('general.image'), log=True) runSystemEx('cp %s VMware_conversion/shared/convert_img.img' % options('general.image'), log=True) convertChannel = threads.runThreadWithChannel( convertImage).channel.sendWithChannel(options) waitForPasswordChange() bundleChannel = threads.runThreadWithChannel( bundleAMI).channel.sendWithChannel(options) try: convertChannel.receive() vmWareDir = 'clovr-vmware.%s' % options('general.version') runSystemEx('mkdir -p ' + vmWareDir, log=True) runSystemEx( 'mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join( vmWareDir, 'clovr.9-04.x86-64.%s.vmdk' % options('general.version'))) runSystemEx('mkdir -p %s %s' % (os.path.join( vmWareDir, 'keys'), os.path.join(vmWareDir, 'user_data')), log=True) runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir, log=True) fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w') clovrConf = config.configFromMap( dict(version=options('general.version'))) for line in open('/usr/local/projects/clovr/start_clovr.vmx'): fout.write(config.replaceStr(line, clovrConf)) except Exception, err: errorPrint('Converting image failed. Error message:') errorPrint(str(err))
def applyOption(val, option, conf): try: # # We want to apply any replacements on the options # The question is if baseConf is really the config file # we should be applying these from... v = option[4](val) # # Now v might still be a function at this point because we want to give # them the ability to access baseConf is they need it to do more replacements v = applyIfCallable(v, conf) try: return replaceStr(v, conf) except TypeError: return v except MissingOptionError, err: raise CLIError(option[0], err)
def runPipelineWithConfig(taskName, name, pipeline, conf, queue): """ This is for internal use only """ templateDir = os.path.join(conf('dirs.clovr_pipelines_template_dir'), pipeline.TEMPLATE_NAME) templateConfig = os.path.join(templateDir, 'pipeline_tmpl.config') templateLayout = os.path.join(templateDir, 'pipeline.layout') foutName = os.path.join('/tmp', str(time.time())) fout = open(foutName, 'w') for line in handleIncludes(open(templateConfig)): fout.write(config.replaceStr(line, conf) + '\n') fout.close() cmd = [ 'run_pipeline.pl', '--config=' + foutName, '--templatelayout=' + templateLayout, '--taskname=' + taskName ] if queue: cmd.append('--queue=' + queue) cmd = ' '.join(cmd) res = [] exitCode = runSingleProgram(cmd, res.append, None) ## # If we got a weird exit code or more than one line was print or nothing was printed # then something bad happened if exitCode != 0 or len(res) > 1 or not res: raise ProgramRunError(cmd, exitCode) ## # This should be the pipeline ID return createPipeline(taskName, name, res[0].strip(), pipeline, conf)
def buildConfigN(options, args=None, usage=None, baseConf=None, putInGeneral=True): """ This builds a config from options. Options is a list of tuples that looks like: (name, short, long, help, func, [bool]) Where name - Name of the option, this is what it will become in the config file short - Short option - needs to start with - long - Long option - needs to start with -- help - Help to be given to a user in --help output func - Function to be applied to the value bool - This is not required, set to True if the option is simply a boolean, all other datatypes can be verified via 'func' This will implicitly check if a 'conf' option exists, and if so load te conf file as a base for these config options. All options are put into the 'general' section. This returns a tuple (conf, args) where args is whatever is left over from parsing This also implicitly loads the current environment into the env section If, when evaluated, 'func' returns a function, it is called with the baseConf. This is to allow more complex replacements to happen. """ def _iterBool(v): """ Adds the non erquired bool field with a default of STRING if it is not present """ for l in v: if len(l) == 6: yield l else: yield tuple(list(l) + [STRING]) parser = optparse.OptionParser(usage=usage) ## # keep track of the function to apply to conf confFunc = None ## # The order of the options below # var name, short option, long option, help message, function to apply, binary option for n, s, l, h, f, b in _iterBool(options): ## # We could have a function we want to apply to the conf variable. We want to store it # so when we use it in the next block we don't have to loop over options looking for it again # This is a minor optimization and probably not even necessary... if n == 'conf': confFunc = f if b == BINARY: parser.add_option(s, l, dest=n, help=h, action='store_true') elif b == LIST: parser.add_option(s, l, dest=n, help=h, action='append') elif b == COUNT: parser.add_option(s, l, dest=n, help=h, action='count') elif b == STRING: parser.add_option(s, l, dest=n, help=h) else: raise Exception('Unknown option type: ' + repr(b)) ops, args = parser.parse_args(args=args) if baseConf is None: baseConf = configFromEnv() if hasattr(ops, 'conf'): baseConf = configFromStream(open(replaceStr(confFunc(ops.conf), baseConf)), baseConf) vals = {} ## # The order of the options below # var name, short option, long option, help message, function to apply, binary option for o in _iterBool(options): n, _s, l, _h, f, _b = o try: vals[n] = applyOption(getattr(ops, n), o, baseConf) except Exception, err: raise CLIError(l, err)
def pkgFileExists(pkgname, fname): """Ensures a file in the package exists""" fname = replaceStr(fname, conf) fileExists(os.path.join(conf('stow.package_dir'), pkgname, fname))
def runSystemExC(conf, cmd): cmd = replaceStr(cmd, conf) runSystemEx(cmd)
def runSingleProgramEx(conf, cmd, stdoutf, stderrf): cmd = replaceStr(cmd, conf) exitCode = runSingleProgram(cmd, stdoutf, stderrf) if exitCode != 0: raise ProgramRunError(cmd, exitCode)
def dirPermissions(dirname, perms, ignoreError=False): dirname = replaceStr(dirname, conf) perms = replaceStr(perms, conf) run('chmod -R %s %s' % (perms, dirname), ignoreError=ignoreError)
def fileExists(fname): fname = replaceStr(fname, conf) if not os.path.exists(fname): raise PolicyError('File does not exist: ' + fname)
def buildConfigN(options, args=None, usage=None, baseConf=None, putInGeneral=True): """ This builds a config from options. Options is a list of tuples that looks like: (name, short, long, help, func, [bool]) Where name - Name of the option, this is what it will become in the config file short - Short option - needs to start with - long - Long option - needs to start with -- help - Help to be given to a user in --help output func - Function to be applied to the value bool - This is not required, set to True if the option is simply a boolean, all other datatypes can be verified via 'func' This will implicitly check if a 'conf' option exists, and if so load te conf file as a base for these config options. All options are put into the 'general' section. This returns a tuple (conf, args) where args is whatever is left over from parsing This also implicitly loads the current environment into the env section If, when evaluated, 'func' returns a function, it is called with the baseConf. This is to allow more complex replacements to happen. """ def _iterBool(v): """ Adds the non erquired bool field with a default of STRING if it is not present """ for l in v: if len(l) == 6: yield l else: yield tuple(list(l) + [STRING]) parser = optparse.OptionParser(usage=usage) ## # keep track of the function to apply to conf confFunc = None ## # The order of the options below # var name, short option, long option, help message, function to apply, binary option for n, s, l, h, f, b in _iterBool(options): ## # We could have a function we want to apply to the conf variable. We want to store it # so when we use it in the next block we don't have to loop over options looking for it again # This is a minor optimization and probably not even necessary... if n == 'conf': confFunc = f if b == BINARY: parser.add_option(s, l, dest=n, help=h, action='store_true') elif b == LIST: parser.add_option(s, l, dest=n, help=h, action='append') elif b == COUNT: parser.add_option(s, l, dest=n, help=h, action='count') elif b == STRING: parser.add_option(s, l, dest=n, help=h) else: raise Exception('Unknown option type: ' + repr(b)) ops, args = parser.parse_args(args=args) if baseConf is None: baseConf = configFromEnv() if hasattr(ops, 'conf'): baseConf = configFromStream( open(replaceStr(confFunc(ops.conf), baseConf)), baseConf) vals = {} ## # The order of the options below # var name, short option, long option, help message, function to apply, binary option for o in _iterBool(options): n, _s, l, _h, f, _b = o try: vals[n] = applyOption(getattr(ops, n), o, baseConf) except Exception, err: raise CLIError(l, err)