def downloadPipeline(instance, conf, pipelineId, outDir, outBaseName, overwrite=False, log=False): """ Downloads a pipeline from an instance. conf - contains various config data, the following options need to be present: ssh.options - options to pass to ssh ssh.user - user to ssh as dirs.clovr_project - the directory on the imave clovr project is expected to live pipelineId - the id of the pipeline to download outDir - local directory to download to outBaseName - The basename for the output file overwrite - if the downloaded file exists locally already, should it be downloaded (default No) """ outF = '/mnt/%s.tar.gz' % outBaseName cmd = ['cd %s;' % conf('dirs.clovr_project'), 'tar', '-zcf', outF, 'output_repository/*/%s_*' % pipelineId, 'workflow/runtime/*/%s_*' % pipelineId] runSystemInstanceEx(instance, ' '.join(cmd), None, (log and errorPrintS or None), user=conf('ssh.user'), options=conf('ssh.options'), log=log) outFilename = os.path.join(outDir, os.path.basename(outF)) fileExists = os.path.exists(outFilename) if fileExists and overwrite or not fileExists: runSystemEx('mkdir -p ' + outDir) scpFromEx(instance['public_dns'], outF, outDir, user=conf('ssh.user'), options=conf('ssh.options'), log=log) runSystemInstanceEx(instance, 'rm ' + outF, None, (log and errorPrintS or None), user=conf('ssh.user'), options=conf('ssh.options'), log=log) return outFilename else: runSystemInstanceEx(instance, 'rm ' + outF, None, (log and errorPrintS or None), user=conf('ssh.user'), options=conf('ssh.options'), log=log) raise DownloadPipelineOverwriteError(outFilename)
def main(options, _args): if options('general.devel'): if not os.path.exists('/mnt/keys/devel1.pem'): runSystemEx('cp %s /mnt/keys' % options('general.devel')) else: if not os.path.exists('/mnt/keys/devel1.pem'): runSingleProgramEx('ssh-keygen -f /mnt/keys/devel1.pem -P ""', None, None) keyData = [] runSingleProgramEx('ssh-keygen -y -f /mnt/keys/devel1.pem', keyData.append, sys.stderr.write) keyData = ''.join(keyData) for path, user in [('/root', 'root'), ('/home/www-data', 'www-data')]: authorizedKeysPath = os.path.join(path, '.ssh', 'authorized_keys') runSingleProgramEx('mkdir -p %s' % os.path.dirname(authorizedKeysPath), None, None) if os.path.exists(authorizedKeysPath): authorizedKeys = open(authorizedKeysPath).read() if keyData not in authorizedKeys: writeKeyData(authorizedKeysPath, keyData, user) else: writeKeyData(authorizedKeysPath, keyData, user) print print print 'Setup complete.' print '*** Remember, currently you have to do this every time you restart the VM'
def addJavaCert(outputdir, host, port): """This installs a java cert. It is assumed that install-cert.sh is in the PATH""" runSystemEx("""cd /tmp/ && echo 1 | install-cert.sh %s:%d""" % (host, port)) if outputdir not in ['/tmp', '/tmp/']: runSystem("""mv /tmp/jssecacerts %s""" % (outputdir, ))
def bundleAMI(chan): options, rchan = chan.receive() try: cmd = ['ec2-bundle-image', '-c ${general.cert}', '-k ${general.key}', '-u ${general.user}', '--kernel ${general.kernel}', '-i ${general.image}', '-d ${general.dest}', '-p ${general.image}', '-r ${general.arch}'] if options('general.ec2cert'): cmd.append('--ec2cert ${general.ec2cert}') runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = ['ec2-upload-bundle', '-b ${general.image}', '-m ${general.dest}/${general.image}.manifest.xml', '-a ${general.access_key}', '-s ${general.secret_access_key}'] runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = ['ec2-register', '${general.image}/${general.image}.manifest.xml', '-K ${general.key}', '-C ${general.cert}'] outp = [] runSingleProgramEx(config.replaceStr(' '.join(cmd), options), stdoutf=outp.append, stderrf=sys.stderr.write, log=True) ami = ''.join(outp).split()[-1] ## # Make the AMI public runSingleProgramEx('ec2-modify-image-attribute %s --launch-permission -a all' % ami, stdoutf=sys.stdout.write, stderrf=sys.stderr.write, log=True) rchan.send(ami) except Exception, err: rchan.sendError(err)
def instantiateCredential(conf, cred): if not conf('config_loaded', default=False): conf = config.configFromMap({'config_loaded': True}, base=config.configFromStream(open( conf('general.conf_file')), base=conf)) certFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert.pem') keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem') if not os.path.exists(certFile) and not os.path.exists(keyFile): tmpCertFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert-tmp.pem') tmpKeyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key-tmp.pem') if 'ec2_url' not in cred.metadata: raise Exception('You must have an ec2_url') parsedUrl = urlparse.urlparse(cred.metadata['ec2_url']) if ':' not in parsedUrl.netloc: raise Exception('Your URL must contain a port') host, port = parsedUrl.netloc.split(':') fout = open(tmpCertFile, 'w') fout.write(cred.cert) fout.close() fout = open(tmpKeyFile, 'w') fout.write(cred.pkey) fout.close() try: commands.runSystemEx(' '.join([ 'nimbusCerts2EC2.py', '--in-cert=' + tmpCertFile, '--out-cert=' + certFile, '--in-key=' + tmpKeyFile, '--out-key=' + keyFile, '--java-cert-dir=/tmp', '--java-cert-host=' + host, '--java-cert-port=' + port ]) + ' > /dev/null 2>&1', log=True) commands.runSystemEx('chmod +r ' + keyFile) finally: os.unlink(tmpCertFile) os.unlink(tmpKeyFile) ec2Home = '/opt/ec2-api-tools-1.3-42584' newCred = func.Record( cert=certFile, pkey=keyFile, ec2Path=os.path.join(ec2Home, 'bin'), env=dict(EC2_JVM_ARGS='-Djavax.net.ssl.trustStore=/tmp/jssecacerts', EC2_HOME=ec2Home, EC2_URL=cred.metadata['ec2_url'])) if os.path.exists(conf('cluster.cluster_private_key') + '.pub'): pubKey = open(conf('cluster.cluster_private_key') + '.pub').read().rstrip() ec2_control.addKeypair(newCred, '"' + conf('cluster.key') + '||' + pubKey + '"') return (conf, newCred)
def sharedFoldersEnabled(): try: ## # Are we running VMWare? runSystemEx('vmware-checkvm > /dev/null 2>&1') res = [] runSingleProgramEx('df', res.append, None) return [l for l in res if l.startswith('.host:/shared')] except: ## # If we aren't running VMWare, just assume it worked for now return True
def convertKey(inname, outname): """ This just calls an openssl command. The user will have to type in their password """ ## # Using os.system here because we want to let the user type in cmd = """openssl rsa -in %s -out %s""" % (inname, outname) code = os.system(cmd) if code != 0: raise ProgramRunError(cmd, code) runSystemEx('chmod 400 ' + outname)
def body(self): form = cgi.FieldStorage() host = form['host'].value sio = StringIO.StringIO() sgeRootBin = os.path.join(SGE_ROOTBIN) if 'ipaddr' in form and re.match('^\d+\.\d+\.\d+\.\d+$', form['ipaddr'].value): commands.runSystemEx('addEtcHosts.py %s %s' % (host, form['ipaddr'].value)) cmd = [os.path.join(sgeRootBin, 'qconf'), '-ah', host, '&>', '/tmp/add_host.out'] commands.runSystemEx(' '.join(cmd))
def checkout(self, options, repo, repoPath, outputPath, branch): fullPath = os.path.join(repo.repoUrl, branch, repoPath) stderr = [] try: # Test to see if it's already checked out, if not continue self._raiseIfCheckout(outputPath) commands.runSingleProgramEx('svn co %s %s' % (fullPath, outputPath), stdoutf=None, stderrf=stderr.append, log=logging.DEBUG) except CheckoutModifiedError: logging.errorPrint( 'You have uncommited changes to %s, please revert changes or commit them' % outputPath) raise except commands.ProgramRunError: if 'refers to a file, not a directory' in ''.join(stderr): try: tmpPath = os.path.dirname( os.path.join(options('general.codir'), repoPath)) self._raiseIfCheckout(tmpPath) commands.runSystem('rm -rf ' + tmpPath, log=logging.DEBUG) commands.runSystemEx('mkdir -p ' + tmpPath, log=logging.DEBUG) commands.runSingleProgramEx( 'svn co %s %s' % (os.path.dirname(fullPath), tmpPath), stdoutf=None, stderrf=logging.errorPrintS, log=logging.DEBUG) commands.runSystem('rm -rf ' + outputPath, log=logging.DEBUG) commands.runSystemEx('ln -s %s %s' % (os.path.join( options('general.codir'), repoPath), outputPath), log=logging.DEBUG) except CheckoutModifiedError: logging.errorPrint( 'You have uncommited changes to %s, please revert changes or commit them' % tmpPath) raise else: for l in stderr: logging.errorPrintS(l) raise logExport(options('general.config_dir'), repo, repoPath, outputPath, branch, CHECKOUT)
def startUpDevNode(conf): """ DEPRECATED: This type is deprecated, instead we should just be using the [dev] section of the config file to specify what branches and what parts of the system to load from svn Steps in starting a dev node: 1 - Remove /usr/local/stow (this should be a config option eventually) 2 - Check out /usr/local/stow 3 - Check out /opt/packages Any SVN work is done on trunk (need to add config to specify a branch) """ executePolicyDirWEx('/opt/config_policies', 'DEV') runSystemEx("""updateAllDirs.py --co""")
def body(self): form = cgi.FieldStorage() host = form['host'].value sio = StringIO.StringIO() sgeRootBin = os.path.join(SGE_ROOTBIN) if 'ipaddr' in form and re.match('^\d+\.\d+\.\d+\.\d+$', form['ipaddr'].value): commands.runSystemEx('addEtcHosts.py %s %s' % (host, form['ipaddr'].value)) cmd = [ os.path.join(sgeRootBin, 'qconf'), '-ah', host, '&>', '/tmp/add_host.out' ] commands.runSystemEx(' '.join(cmd))
def bundleAMI(chan): options, rchan = chan.receive() try: cmd = [ 'ec2-bundle-image', '-c ${general.cert}', '-k ${general.key}', '-u ${general.user}', '--kernel ${general.kernel}', '-i ${general.image}', '-d ${general.dest}', '-p ${general.image}', '-r ${general.arch}' ] if options('general.ec2cert'): cmd.append('--ec2cert ${general.ec2cert}') runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = [ 'ec2-upload-bundle', '-b ${general.image}', '-m ${general.dest}/${general.image}.manifest.xml', '-a ${general.access_key}', '-s ${general.secret_access_key}' ] runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = [ 'ec2-register', '${general.image}/${general.image}.manifest.xml', '-K ${general.key}', '-C ${general.cert}' ] outp = [] runSingleProgramEx(config.replaceStr(' '.join(cmd), options), stdoutf=outp.append, stderrf=sys.stderr.write, log=True) ami = ''.join(outp).split()[-1] ## # Make the AMI public runSingleProgramEx( 'ec2-modify-image-attribute %s --launch-permission -a all' % ami, stdoutf=sys.stdout.write, stderrf=sys.stderr.write, log=True) rchan.send(ami) except Exception, err: rchan.sendError(err)
def runTaskStatus(taskName, clusterName=None, hostName='localhost'): """ This is a simple function that takes a taskname and simply runs vp-describe-task on it. It is meant to be used in specific situations in front ends. It is not meant to be a generic function. There are no guarantees that this funciton will exist tomorrow and it could be moved into a more fitting location at any point """ cmd = [ 'vp-describe-task', '--show', '--show-error', '--exit-code', '--block', '--no-print-polling' ] if clusterName: cmd.append('--name=' + clusterName) cmd.append('--host=' + hostName) cmd.append(taskName) commands.runSystemEx(' '.join(cmd))
def instantiateCredential(conf, cred): if not conf('config_loaded', default=False): conf = config.configFromMap({'config_loaded': True}, base=config.configFromStream(open(conf('general.conf_file')), base=conf)) certFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert.pem') keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem') if not os.path.exists(certFile) and not os.path.exists(keyFile): tmpCertFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert-tmp.pem') tmpKeyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key-tmp.pem') if 'ec2_url' not in cred.metadata: raise Exception('You must have an ec2_url') parsedUrl = urlparse.urlparse(cred.metadata['ec2_url']) if ':' not in parsedUrl.netloc: raise Exception('Your URL must contain a port') host, port = parsedUrl.netloc.split(':') fout = open(tmpCertFile, 'w') fout.write(cred.cert) fout.close() fout = open(tmpKeyFile, 'w') fout.write(cred.pkey) fout.close() try: commands.runSystemEx(' '.join(['nimbusCerts2EC2.py', '--in-cert=' + tmpCertFile, '--out-cert=' + certFile, '--in-key=' + tmpKeyFile, '--out-key=' + keyFile, '--java-cert-dir=/tmp', '--java-cert-host=' + host, '--java-cert-port=' + port]) + ' > /dev/null 2>&1', log=True) commands.runSystemEx('chmod +r ' + keyFile) finally: os.unlink(tmpCertFile) os.unlink(tmpKeyFile) ec2Home = '/opt/ec2-api-tools-1.3-42584' newCred = func.Record(cert=certFile, pkey=keyFile, ec2Path=os.path.join(ec2Home, 'bin'), env=dict(EC2_JVM_ARGS='-Djavax.net.ssl.trustStore=/tmp/jssecacerts', EC2_HOME=ec2Home, EC2_URL=cred.metadata['ec2_url'])) if os.path.exists(conf('cluster.cluster_private_key') + '.pub'): pubKey = open(conf('cluster.cluster_private_key') + '.pub').read().rstrip() ec2_control.addKeypair(newCred, '"' + conf('cluster.key') + '||' + pubKey + '"') return (conf, newCred)
def runTaskStatus(taskName, clusterName=None, hostName='localhost'): """ This is a simple function that takes a taskname and simply runs vp-describe-task on it. It is meant to be used in specific situations in front ends. It is not meant to be a generic function. There are no guarantees that this funciton will exist tomorrow and it could be moved into a more fitting location at any point """ cmd = ['vp-describe-task', '--show', '--show-error', '--exit-code', '--block', '--no-print-polling'] if clusterName: cmd.append('--name=' + clusterName) cmd.append('--host=' + hostName) cmd.append(taskName) commands.runSystemEx(' '.join(cmd))
def startUpAllNodes(conf): """ Things that need to be done for all nodes: 1 - Go through /usr/local/stow and /opt/opt-packages installing all packages 2 - List through all .py files in /opt/config_policies and run them (eventually this directory should probably be split up by node type) """ if conf('dev.update_dirs', default=None): cmd = [ 'updateAllDirs.py', '--clovr-branch=' + conf('dev.clovr_branch', default=''), '--vappio-branch=' + conf('dev.vappio_branch', default=''), conf('dev.update_dirs') ] runSystemEx(' '.join(cmd)) installAllStow() installAllOptPackages() executePolicyDirWEx(startPolicy, '/opt/config_policies')
def main(options, args): logging.DEBUG = options('general.debug') urls = [] if not args: for line in [l for l in sys.stdin if l.strip()]: urls.append(line.strip() + options('general.suffix')) else: urls.extend([u + options('general.suffix') for u in args]) for url in urls: reliableDownloader.deleteDownloadedFiles('/tmp', url) commands.runSystemEx(' '.join(['wget', '-P', '/tmp', '-nv', url]), log=logging.DEBUG) for f in reliableDownloader.getDownloadFilenames('/tmp', url): sys.stdout.write('\n'.join([ '%s %s' % (l.strip().split()[0], os.path.join(os.path.dirname(url), l.strip().split()[1])) for l in open(f) ]) + '\n') reliableDownloader.deleteDownloadedFiles('/tmp', url)
def startUpAllNodes(conf): """ Things that need to be done for all nodes: 1 - Go through /usr/local/stow and /opt/opt-packages installing all packages 2 - List through all .py files in /opt/config_policies and run them (eventually this directory should probably be split up by node type) """ if conf('dev.update_dirs', default=None): cmd = ['updateAllDirs.py', '--clovr-branch=' + conf('dev.clovr_branch', default=''), '--vappio-branch=' + conf('dev.vappio_branch', default=''), conf('dev.update_dirs')] runSystemEx(' '.join(cmd)) installAllStow() installAllOptPackages() executePolicyDirWEx(startPolicy, '/opt/config_policies')
def grabFromSVN(options, srcUrl, branch, d, dstDir): cmd = ['echo p | svn'] if options('general.co'): cmd += ['co'] else: cmd += ['export', '--force'] cmd += [srcUrl + '/' + branch + '/' + d, dstDir] outp = [] runSingleProgramEx('svn status ' + dstDir, stdoutf=outp.append, stderrf=None, log=False) ## # If outp contains some output it means modifications have been made if outp: raise CheckoutModifiedError( 'There are modifications to %s, please commit them or revert them before continuing' % dstDir) runSystemEx('rm -rf ' + dstDir, log=True) runSystemEx(' '.join(cmd), log=True)
def checkout(self, options, repo, repoPath, outputPath, branch): fullPath = os.path.join(repo.repoUrl, branch, repoPath) stderr = [] try: # Test to see if it's already checked out, if not continue self._raiseIfCheckout(outputPath) commands.runSingleProgramEx('svn co %s %s' % (fullPath, outputPath), stdoutf=None, stderrf=stderr.append, log=logging.DEBUG) except CheckoutModifiedError: logging.errorPrint('You have uncommited changes to %s, please revert changes or commit them' % outputPath) raise except commands.ProgramRunError: if 'refers to a file, not a directory' in ''.join(stderr): try: tmpPath = os.path.dirname(os.path.join(options('general.codir'), repoPath)) self._raiseIfCheckout(tmpPath) commands.runSystem('rm -rf ' + tmpPath, log=logging.DEBUG) commands.runSystemEx('mkdir -p ' + tmpPath, log=logging.DEBUG) commands.runSingleProgramEx('svn co %s %s' % (os.path.dirname(fullPath), tmpPath), stdoutf=None, stderrf=logging.errorPrintS, log=logging.DEBUG) commands.runSystem('rm -rf ' + outputPath, log=logging.DEBUG) commands.runSystemEx('ln -s %s %s' % (os.path.join(options('general.codir'), repoPath), outputPath), log=logging.DEBUG) except CheckoutModifiedError: logging.errorPrint('You have uncommited changes to %s, please revert changes or commit them' % tmpPath) raise else: for l in stderr: logging.errorPrintS(l) raise logExport(options('general.config_dir'), repo, repoPath, outputPath, branch, CHECKOUT)
def main(options, _args): cmd = ['ec2-bundle-image', '-c ${general.cert}', '-k ${general.key}', '-u ${general.user}', '--kernel ${general.kernel}', '-i ${general.image}', '-d ${general.dest}', '-p ${general.image}', '-r ${general.arch}'] if options('general.ec2cert'): cmd.append('--ec2cert ${general.ec2cert}') runSystemEx(replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = ['ec2-upload-bundle', '-b ${general.image}', '-m ${general.dest}/${general.image}.manifest.xml', '-a ${general.access_key}', '-s ${general.secret_access_key}'] runSystemEx(replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = ['ec2-register', '${general.image}/${general.image}.manifest.xml', '-K ${general.key}', '-C ${general.cert}'] ## # We want to output the AMI regardless runSystemEx(replaceStr(' '.join(cmd), options), log=True)
def main(options, _args): open('/opt/CLOVR-RELEASE', 'w').write(options('general.version_string') + '\n') for c in COMMANDS: runSystemEx(c, log=True)
def main(options, _args): runSystemEx('svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx('svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx('scp %s:/export/%s .' % (options('general.remote_name'), options('general.image')), log=True) runSystemEx('cp %s /usr/local/projects/clovr/images' % options('general.image'), log=True) runSystemEx('cp %s VMware_conversion/shared/convert_img.img' % options('general.image'), log=True) convertChannel = threads.runThreadWithChannel(convertImage).channel.sendWithChannel(options) waitForPasswordChange() bundleChannel = threads.runThreadWithChannel(bundleAMI).channel.sendWithChannel(options) try: convertChannel.receive() vmWareDir = 'clovr-vmware.%s' % options('general.version') runSystemEx('mkdir -p ' + vmWareDir, log=True) runSystemEx('mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join(vmWareDir, 'clovr.9-04.x86-64.%s.vmdk' % options('general.version'))) runSystemEx('mkdir -p %s %s' % (os.path.join(vmWareDir, 'keys'), os.path.join(vmWareDir, 'user_data')), log=True) runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir, log=True) fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w') clovrConf = config.configFromMap(dict(version=options('general.version'))) for line in open('/usr/local/projects/clovr/start_clovr.vmx'): fout.write(config.replaceStr(line, clovrConf)) except Exception, err: errorPrint('Converting image failed. Error message:') errorPrint(str(err))
def runCommand(_ctype, _baseDir, command, _tagfile): runSystemEx(command)
def tagData(tagsDir, tagName, tagBaseDir, files, recursive, expand, compress, append, overwrite, metadata=None, filterF=None): """ Tag a list of files with the name. The files can contain direcotires, and if recursive is set the contends of the directories will become part of the tag rather than just the name tagBaseDir is the name of the directory that is not part of the actual tag heirarchy expand will cause any archives listed to be expanded and the contents of the archive to be added compress will compress the files that have been put in the tag. compress should be the path to the directory the compressed file should be put. append will add to a tagName if it already exists, only unique names will be kept though filterF - if you want to filter any of the files as they are added to the file list provide a filter function that will be called on each individual file name. The file will be added if filter returns True This returns the tag that was created """ if metadata is None: metadata = {} if not os.path.exists(tagsDir): runSystemEx('mkdir -p ' + tagsDir) outName = os.path.join(tagsDir, tagName) if os.path.exists(outName) and not append and not overwrite: raise Exception('Tag already exists') ## # Keep a set of all old entries in the file, when we walk the generator we'll # we'll check to see if the file already exists in here if append and os.path.exists(outName): oldFiles = set([l.strip() for l in open(outName)]) else: oldFiles = set() files = [ f for f in generateFileList(files, recursive, expand) if f not in oldFiles and (not filterF or filterF and filterF(f)) ] if overwrite: ## # If we are just overwritign the file, no need to old the list of oldFiles # Technically it shouldn't matter but if the old file list is really large # the lookup could be expensive outFile = open(outName, 'w') oldFiles = set() else: outFile = open(outName, 'a') outFile.write('\n'.join(files)) outFile.write('\n') outFile.close() # # If we are compressing the files then, load the tag back up # so we have all of the files there if compress: outTar = str(os.path.join(compress, tagName + '.tar')) outGzip = outTar + '.gz' if os.path.exists(outGzip): os.remove(outGzip) runSystemEx('mkdir -p ' + compress) files = loadTagFile(outName)('files') baseDirFiles, nonBaseDirFiles = partitionFiles(files, tagBaseDir) if baseDirFiles: for fs in func.chunk(20, baseDirFiles): cmd = [ 'tar', '-C', tagBaseDir, '-rf', outTar, ] cmd.extend([removeBase('/', f) for f in fs]) runSystemEx(' '.join(cmd), log=True) if nonBaseDirFiles: for fs in func.chunk(20, nonBaseDirFiles): cmd = [ 'tar', '-C', '/', '-rf', outTar, ] cmd.extend([removeBase('/', f) for f in fs]) runSystemEx(' '.join(cmd), log=True) # # It's possible we have no values here, if so, the tar was not created # and should be ignored if os.path.exists(outTar): runSystemEx('gzip ' + outTar, log=True) metadata = func.updateDict(metadata, { 'compressed': True, 'compressed_file': outGzip }) # # If tagBaseDir is set it means we have some metadata to write if tagBaseDir: metadata['tag_base_dir'] = tagBaseDir if append and os.path.exists(outName + '.metadata'): tmd = json.loads(open(outName + '.metadata').read()) metadata = func.updateDict(tmd, metadata) outFile = open(outName + '.metadata', 'w') outFile.write(json.dumps(metadata, indent=1) + '\n') outFile.close() return loadTagFile(outName)
def main(options, _args): logging.DEBUG = options('debug') env = {'PATH': os.path.join(options('prefix'), 'bin') + ':' + os.getenv('PATH')} tmpDir = os.path.join(options('tmpdir'), 'ca-ocaml-platform') buildDir = os.path.join(tmpDir, 'build') commands.runSystem('rm -rf ' + tmpDir, log=logging.DEBUG) commands.runSystemEx('mkdir -p ' + tmpDir, log=logging.DEBUG) commands.runSystemEx('mkdir -p ' + buildDir, log=logging.DEBUG) # # Download everything first for _, url in URLS: commands.runSystemEx('wget --quiet -P %s %s' % (tmpDir, url), log=logging.DEBUG) for project, url in URLS: fileName = os.path.basename(url) downloadedName = os.path.join(tmpDir, fileName) untarToDir(downloadedName, buildDir) # Ocaml gets special attention because it needs the prefix if project == 'ocaml': commands.runSingleProgramEx('cd %s/*%s*; ./configure -prefix %s' % (buildDir, project, options('prefix')), stdoutf=None, stderrf=sys.stderr.write, log=logging.DEBUG) commands.runSingleProgramEx('cd %s/*%s*; make world.opt install' % (buildDir, project), stdoutf=None, stderrf=sys.stderr.write, log=logging.DEBUG) elif project in ['findlib']: commands.runSingleProgramEx('cd %s/*%s*; ./configure' % (buildDir, project), stdoutf=None, stderrf=sys.stderr.write, addEnv=env, log=logging.DEBUG) commands.runSingleProgramEx('cd %s/*%s*; make all opt install' % (buildDir, project), stdoutf=None, stderrf=sys.stderr.write, addEnv=env, log=logging.DEBUG) elif project == 'core': commands.runSingleProgramEx('cd %s/*%s*; patch -p1 -i %s/patches/core-0.6.0-3.12.0.patch' % (buildDir, project, os.getcwd()), stdoutf=None, stderrf=sys.stderr.write, log=logging.DEBUG) commands.runSingleProgramEx('cd %s/*%s*; make' % (buildDir, project), stdoutf=None, stderrf=sys.stderr.write, addEnv=env, log=logging.DEBUG) commands.runSingleProgramEx('cd %s/*%s*; make install' % (buildDir, project), stdoutf=None, stderrf=sys.stderr.write, addEnv=env, log=logging.DEBUG) else: commands.runSingleProgramEx('cd %s/*%s*; make' % (buildDir, project), stdoutf=None, stderrf=sys.stderr.write, addEnv=env, log=logging.DEBUG) commands.runSingleProgramEx('cd %s/*%s*; make install' % (buildDir, project), stdoutf=None, stderrf=sys.stderr.write, addEnv=env, log=logging.DEBUG) commands.runSystemEx('rm -rf %s/*' % buildDir, log=logging.DEBUG) print 'Be sure to update your environment variables:' print 'export PATH=%s:$PATH' % (os.path.join(options('prefix'), 'bin'),)
def main(options, _args): runSystemEx( 'svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx( 'svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx('scp %s:/export/%s .' % (options('general.remote_name'), options('general.image')), log=True) runSystemEx('cp %s /usr/local/projects/clovr/images' % options('general.image'), log=True) runSystemEx('cp %s VMware_conversion/shared/convert_img.img' % options('general.image'), log=True) convertChannel = threads.runThreadWithChannel( convertImage).channel.sendWithChannel(options) waitForPasswordChange() bundleChannel = threads.runThreadWithChannel( bundleAMI).channel.sendWithChannel(options) try: convertChannel.receive() vmWareDir = 'clovr-vmware.%s' % options('general.version') runSystemEx('mkdir -p ' + vmWareDir, log=True) runSystemEx( 'mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join( vmWareDir, 'clovr.9-04.x86-64.%s.vmdk' % options('general.version'))) runSystemEx('mkdir -p %s %s' % (os.path.join( vmWareDir, 'keys'), os.path.join(vmWareDir, 'user_data')), log=True) runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir, log=True) fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w') clovrConf = config.configFromMap( dict(version=options('general.version'))) for line in open('/usr/local/projects/clovr/start_clovr.vmx'): fout.write(config.replaceStr(line, clovrConf)) except Exception, err: errorPrint('Converting image failed. Error message:') errorPrint(str(err))
def runSystemExC(conf, cmd): cmd = replaceStr(cmd, conf) runSystemEx(cmd)
def main(options, _args): updateAll = False for o in OPTIONS: if o[0] not in ['vappio_branch', 'clovr_branch' ] and options('general.' + o[0]): break else: updateAll = True clovrBranch = options('general.clovr_branch') vappioBranch = options('general.vappio_branch') try: if options('general.stow') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code', clovrBranch, 'stow', '/usr/local/stow') if options('general.opt_packages') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code', clovrBranch, 'opt-packages', '/opt/opt-packages') if options('general.config_policies') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code', clovrBranch, 'config_policies', '/opt/config_policies') if options('general.vappio_py') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code', vappioBranch, 'vappio-py', '/opt/vappio-py') runSystemEx("""chmod +x /opt/vappio-py/vappio/cli/*.py""") if options('general.vappio_apps') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code', vappioBranch, 'vappio-apps', '/opt/vappio-apps') if options('general.vappio_twisted') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code', vappioBranch, 'vappio-twisted', '/opt/vappio-twisted') if options('general.vappio_scripts') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code', vappioBranch, 'vappio-scripts', '/opt/vappio-scripts') runSystemEx("""chmod -R +x /opt/vappio-scripts""", log=True) runSystemEx("""cp -f /opt/vappio-scripts/clovrEnv.sh /root""", log=True) runSystemEx( """cp -f /opt/vappio-scripts/local /etc/init.d/local""", log=True) runSystemEx( """cp -f /opt/vappio-scripts/rc.local /etc/init.d/rc.local""", log=True) runSystemEx( """cp -f /opt/vappio-scripts/screenrc /root/.screenrc""", log=True) if options('general.clovr_pipelines') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code', clovrBranch, 'clovr_pipelines', '/opt/clovr_pipelines') if options('general.vappio_py_www') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code', vappioBranch, 'vappio-www/py-www', '/var/www/vappio') ## # Only want to do this one when specified # if options('general.vappio_conf'): # grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code', vappioBranch, 'vappio-conf', '/mnt/vappio-conf') if options('general.hudson') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code', clovrBranch, 'hudson/hudson-config/jobs', '/var/lib/hudson/jobs') grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code', clovrBranch, 'hudson/hudson-scripts', '/opt/hudson') runSystemEx("""chown -R hudson.nogroup /var/lib/hudson/jobs""", log=True) if options('general.clovr_www') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code', clovrBranch, 'clovr-www', '/var/www/clovr') except CheckoutModifiedError, err: errorPrint(str(err))
def tagData(tagsDir, tagName, tagBaseDir, files, recursive, expand, compress, append, overwrite, metadata=None, filterF=None): """ Tag a list of files with the name. The files can contain direcotires, and if recursive is set the contends of the directories will become part of the tag rather than just the name tagBaseDir is the name of the directory that is not part of the actual tag heirarchy expand will cause any archives listed to be expanded and the contents of the archive to be added compress will compress the files that have been put in the tag. compress should be the path to the directory the compressed file should be put. append will add to a tagName if it already exists, only unique names will be kept though filterF - if you want to filter any of the files as they are added to the file list provide a filter function that will be called on each individual file name. The file will be added if filter returns True This returns the tag that was created """ if metadata is None: metadata = {} if not os.path.exists(tagsDir): runSystemEx('mkdir -p ' + tagsDir) outName = os.path.join(tagsDir, tagName) if os.path.exists(outName) and not append and not overwrite: raise Exception('Tag already exists') ## # Keep a set of all old entries in the file, when we walk the generator we'll # we'll check to see if the file already exists in here if append and os.path.exists(outName): oldFiles = set([l.strip() for l in open(outName)]) else: oldFiles = set() files = [f for f in generateFileList(files, recursive, expand) if f not in oldFiles and (not filterF or filterF and filterF(f))] if overwrite: ## # If we are just overwritign the file, no need to old the list of oldFiles # Technically it shouldn't matter but if the old file list is really large # the lookup could be expensive outFile = open(outName, 'w') oldFiles = set() else: outFile = open(outName, 'a') outFile.write('\n'.join(files)) outFile.write('\n') outFile.close() # # If we are compressing the files then, load the tag back up # so we have all of the files there if compress: outTar = str(os.path.join(compress, tagName + '.tar')) outGzip = outTar + '.gz' if os.path.exists(outGzip): os.remove(outGzip) runSystemEx('mkdir -p ' + compress) files = loadTagFile(outName)('files') baseDirFiles, nonBaseDirFiles = partitionFiles(files, tagBaseDir) if baseDirFiles: for fs in func.chunk(20, baseDirFiles): cmd = ['tar', '-C', tagBaseDir, '-rf', outTar, ] cmd.extend([removeBase('/', f) for f in fs]) runSystemEx(' '.join(cmd), log=True) if nonBaseDirFiles: for fs in func.chunk(20, nonBaseDirFiles): cmd = ['tar', '-C', '/', '-rf', outTar, ] cmd.extend([removeBase('/', f) for f in fs]) runSystemEx(' '.join(cmd), log=True) # # It's possible we have no values here, if so, the tar was not created # and should be ignored if os.path.exists(outTar): runSystemEx('gzip ' + outTar, log=True) metadata = func.updateDict(metadata, {'compressed': True, 'compressed_file': outGzip}) # # If tagBaseDir is set it means we have some metadata to write if tagBaseDir: metadata['tag_base_dir'] = tagBaseDir if append and os.path.exists(outName + '.metadata'): tmd = json.loads(open(outName + '.metadata').read()) metadata = func.updateDict(tmd, metadata) outFile = open(outName + '.metadata', 'w') outFile.write(json.dumps(metadata, indent=1) + '\n') outFile.close() return loadTagFile(outName)
def addJavaCert(outputdir, host, port): """This installs a java cert. It is assumed that install-cert.sh is in the PATH""" runSystemEx("""cd /tmp/ && echo 1 | install-cert.sh %s:%d""" % (host, port)) if outputdir not in ['/tmp', '/tmp/']: runSystem("""mv /tmp/jssecacerts %s""" % (outputdir,))