def getSizeOfFiles(files): stdoutL = [] commands.runSingleProgramEx('du -kcs ' + ' '.join(files), stdoutf=stdoutL.append, stderrf=None, log=False) return int(stdoutL[-1].split()[0])
def writeKeyData(authKeysPath, keyData, user): fout = open(authKeysPath, 'a') fout.write('\n' + keyData + '\n') fout.close() runSingleProgramEx('chown %s:%s %s' % (user, user, authKeysPath), None, None) runSingleProgramEx('chmod 600 ' + authKeysPath, None, None)
def convertImage(chan): options, rchan = chan.receive() try: runSingleProgramEx('vmplayer VMware_conversion/conversion_image.vmx', stdoutf=None, stderrf=None, log=True) rchan.send(None) except Exception, err: rchan.sendError(err)
def bundleAMI(chan): options, rchan = chan.receive() try: cmd = ['ec2-bundle-image', '-c ${general.cert}', '-k ${general.key}', '-u ${general.user}', '--kernel ${general.kernel}', '-i ${general.image}', '-d ${general.dest}', '-p ${general.image}', '-r ${general.arch}'] if options('general.ec2cert'): cmd.append('--ec2cert ${general.ec2cert}') runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = ['ec2-upload-bundle', '-b ${general.image}', '-m ${general.dest}/${general.image}.manifest.xml', '-a ${general.access_key}', '-s ${general.secret_access_key}'] runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = ['ec2-register', '${general.image}/${general.image}.manifest.xml', '-K ${general.key}', '-C ${general.cert}'] outp = [] runSingleProgramEx(config.replaceStr(' '.join(cmd), options), stdoutf=outp.append, stderrf=sys.stderr.write, log=True) ami = ''.join(outp).split()[-1] ## # Make the AMI public runSingleProgramEx('ec2-modify-image-attribute %s --launch-permission -a all' % ami, stdoutf=sys.stdout.write, stderrf=sys.stderr.write, log=True) rchan.send(ami) except Exception, err: rchan.sendError(err)
def main(options, _args): res = [] runSingleProgramEx('hostname -f', res.append, errorPrintS) localHost = ''.join(res).strip() outFile = createDataFile(options, [options('general.mode')], options('general.output')) open(outFile, 'a').write('MASTER_IP=' + localHost + '\n')
def createMasterDataFile(cluster, machineConf): """ Creates a master data file as the perl start_cluster works """ template = open(cluster.config('cluster.master_user_data_tmpl')).read() clusterPrivateKey = open( cluster.config('cluster.cluster_private_key')).read() outf = [] runSingleProgramEx('ssh-keygen -y -f ' + cluster.config('cluster.cluster_private_key'), outf.append, None, log=logging.DEBUG) clusterPublicKey = ''.join(outf) template = template.replace('<TMPL_VAR NAME=CLUSTER_PRIVATE_KEY>', clusterPrivateKey) template = template.replace('<TMPL_VAR NAME=CLUSTER_PUBLIC_KEY>', clusterPublicKey) # Need to escape the ${ for bash template = template.replace('<TMPL_VAR NAME=MACHINE_CONF>', open(machineConf).read().replace('${', '\\${')) outf = os.path.join(cluster.config('general.secure_tmp'), 'master_user_data.%s.sh' % global_state.make_ref()) open(outf, 'w').write(template) return outf
def export(self, options, repo, repoPath, outputPath, branch): fullPath = os.path.join(repo.repoUrl, branch, repoPath) self._raiseIfCheckout(outputPath) commands.runSingleProgramEx('echo p | svn export --force %s %s' % (fullPath, outputPath), stdoutf=None, stderrf=logging.errorPrintS, log=logging.DEBUG) logExport(options('general.config_dir'), repo, repoPath, outputPath, branch, EXPORT)
def _raiseIfCheckout(self, path): outp = [] commands.runSingleProgramEx('svn status ' + path, stdoutf=outp.append, stderrf=None, log=False) # If outp contains some output it means modifications have been made if outp: raise CheckoutModifiedError( 'There are modifications to %s, please commit them or revert them before continuing' % path)
def createExecDataFile(conf, master, masterMachineConf): """ Creates a exec data file as the perl start_cluster works This is very similar to createMasterDataFile, should be refactored a bit """ outName = os.path.join('/tmp', str(time.time())) ## # Going to load the master machine.conf and modify node type masterConf = config.configFromStream(open(masterMachineConf), lazy=True) masterConf = config.configFromMap({'NODE_TYPE': EXEC_NODE}, masterConf, lazy=True) fout = open(outName, 'w') fout.write('\n'.join([ k + '=' + str(v) for k, v in config.configToDict(masterConf).iteritems() ])) fout.close() template = open(conf('cluster.exec_user_data_tmpl')).read() clusterPrivateKey = open(conf('cluster.cluster_private_key')).read() outf = [] runSingleProgramEx('ssh-keygen -y -f ' + conf('cluster.cluster_private_key'), outf.append, None, log=True) if conf('general.ctype') == 'ec2': template = template.replace('<TMPL_VAR NAME=MASTER_DNS>', master['private_dns']) else: template = template.replace('<TMPL_VAR NAME=MASTER_DNS>', master['public_dns']) clusterPublicKey = ''.join(outf) template = template.replace('<TMPL_VAR NAME=CLUSTER_PRIVATE_KEY>', clusterPrivateKey) template = template.replace('<TMPL_VAR NAME=CLUSTER_PUBLIC_KEY>', clusterPublicKey) template = template.replace('<TMPL_VAR NAME=MACHINE_CONF>', open(outName).read().replace('${', '\\${')) os.remove(outName) outf = os.path.join(conf('general.secure_tmp'), 'exec_user_data.sh') open(outf, 'w').write(template) return outf
def sharedFoldersEnabled(): try: ## # Are we running VMWare? runSystemEx('vmware-checkvm > /dev/null 2>&1') res = [] runSingleProgramEx('df', res.append, None) return [l for l in res if l.startswith('.host:/shared')] except: ## # If we aren't running VMWare, just assume it worked for now return True
def checkout(self, options, repo, repoPath, outputPath, branch): fullPath = os.path.join(repo.repoUrl, branch, repoPath) stderr = [] try: # Test to see if it's already checked out, if not continue self._raiseIfCheckout(outputPath) commands.runSingleProgramEx('svn co %s %s' % (fullPath, outputPath), stdoutf=None, stderrf=stderr.append, log=logging.DEBUG) except CheckoutModifiedError: logging.errorPrint( 'You have uncommited changes to %s, please revert changes or commit them' % outputPath) raise except commands.ProgramRunError: if 'refers to a file, not a directory' in ''.join(stderr): try: tmpPath = os.path.dirname( os.path.join(options('general.codir'), repoPath)) self._raiseIfCheckout(tmpPath) commands.runSystem('rm -rf ' + tmpPath, log=logging.DEBUG) commands.runSystemEx('mkdir -p ' + tmpPath, log=logging.DEBUG) commands.runSingleProgramEx( 'svn co %s %s' % (os.path.dirname(fullPath), tmpPath), stdoutf=None, stderrf=logging.errorPrintS, log=logging.DEBUG) commands.runSystem('rm -rf ' + outputPath, log=logging.DEBUG) commands.runSystemEx('ln -s %s %s' % (os.path.join( options('general.codir'), repoPath), outputPath), log=logging.DEBUG) except CheckoutModifiedError: logging.errorPrint( 'You have uncommited changes to %s, please revert changes or commit them' % tmpPath) raise else: for l in stderr: logging.errorPrintS(l) raise logExport(options('general.config_dir'), repo, repoPath, outputPath, branch, CHECKOUT)
def runWithCred(cred, cmd, stdoutf=logging.OUTSTREAM.write, stderrf=logging.ERRSTREAM.write, log=False): cmdPrefix = "" if hasattr(cred, "ec2Path"): cmdPrefix = cred.ec2Path + "/" return commands.runSingleProgramEx( cmdPrefix + " ".join(addCredInfo(cmd, cred)), stdoutf, stderrf, log=log, addEnv=cred.env )
def createExecDataFile(conf, master, masterMachineConf): """ Creates a exec data file as the perl start_cluster works This is very similar to createMasterDataFile, should be refactored a bit """ outName = os.path.join('/tmp', str(time.time())) ## # Going to load the master machine.conf and modify node type masterConf = config.configFromStream(open(masterMachineConf), lazy=True) masterConf = config.configFromMap({'NODE_TYPE': EXEC_NODE}, masterConf, lazy=True) fout = open(outName, 'w') fout.write('\n'.join([k + '=' + str(v) for k, v in config.configToDict(masterConf).iteritems()])) fout.close() template = open(conf('cluster.exec_user_data_tmpl')).read() clusterPrivateKey = open(conf('cluster.cluster_private_key')).read() outf = [] runSingleProgramEx('ssh-keygen -y -f ' + conf('cluster.cluster_private_key'), outf.append, None, log=True) if conf('general.ctype') == 'ec2': template = template.replace('<TMPL_VAR NAME=MASTER_DNS>', master['private_dns']) else: template = template.replace('<TMPL_VAR NAME=MASTER_DNS>', master['public_dns']) clusterPublicKey = ''.join(outf) template = template.replace('<TMPL_VAR NAME=CLUSTER_PRIVATE_KEY>', clusterPrivateKey) template = template.replace('<TMPL_VAR NAME=CLUSTER_PUBLIC_KEY>', clusterPublicKey) template = template.replace('<TMPL_VAR NAME=MACHINE_CONF>', open(outName).read().replace('${', '\\${')) os.remove(outName) outf = os.path.join(conf('general.secure_tmp'), 'exec_user_data.sh') open(outf, 'w').write(template) return outf
def bundleAMI(chan): options, rchan = chan.receive() try: cmd = [ 'ec2-bundle-image', '-c ${general.cert}', '-k ${general.key}', '-u ${general.user}', '--kernel ${general.kernel}', '-i ${general.image}', '-d ${general.dest}', '-p ${general.image}', '-r ${general.arch}' ] if options('general.ec2cert'): cmd.append('--ec2cert ${general.ec2cert}') runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = [ 'ec2-upload-bundle', '-b ${general.image}', '-m ${general.dest}/${general.image}.manifest.xml', '-a ${general.access_key}', '-s ${general.secret_access_key}' ] runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug')) cmd = [ 'ec2-register', '${general.image}/${general.image}.manifest.xml', '-K ${general.key}', '-C ${general.cert}' ] outp = [] runSingleProgramEx(config.replaceStr(' '.join(cmd), options), stdoutf=outp.append, stderrf=sys.stderr.write, log=True) ami = ''.join(outp).split()[-1] ## # Make the AMI public runSingleProgramEx( 'ec2-modify-image-attribute %s --launch-permission -a all' % ami, stdoutf=sys.stdout.write, stderrf=sys.stderr.write, log=True) rchan.send(ami) except Exception, err: rchan.sendError(err)
def rsyncFromEx(host, src, dst, rsyncOptions=None, user=None, log=False): cmd = ['rsync'] if rsyncOptions: cmd.append(rsyncOptions) if user: host = user + '@' + host cmd.extend([host + ':' + src, dst]) return commands.runSingleProgramEx(' '.join(cmd), stdoutf=None, stderrf=errorPrintS, log=log)
def main(options, _args): if options('general.devel'): if not os.path.exists('/mnt/keys/devel1.pem'): runSystemEx('cp %s /mnt/keys' % options('general.devel')) else: if not os.path.exists('/mnt/keys/devel1.pem'): runSingleProgramEx('ssh-keygen -f /mnt/keys/devel1.pem -P ""', None, None) keyData = [] runSingleProgramEx('ssh-keygen -y -f /mnt/keys/devel1.pem', keyData.append, sys.stderr.write) keyData = ''.join(keyData) for path, user in [('/root', 'root'), ('/home/www-data', 'www-data')]: authorizedKeysPath = os.path.join(path, '.ssh', 'authorized_keys') runSingleProgramEx('mkdir -p %s' % os.path.dirname(authorizedKeysPath), None, None) if os.path.exists(authorizedKeysPath): authorizedKeys = open(authorizedKeysPath).read() if keyData not in authorizedKeys: writeKeyData(authorizedKeysPath, keyData, user) else: writeKeyData(authorizedKeysPath, keyData, user) print print print 'Setup complete.' print '*** Remember, currently you have to do this every time you restart the VM'
def grabFromSVN(options, srcUrl, branch, d, dstDir): cmd = ['echo p | svn'] if options('general.co'): cmd += ['co'] else: cmd += ['export', '--force'] cmd += [srcUrl + '/' + branch + '/' + d, dstDir] outp = [] runSingleProgramEx('svn status ' + dstDir, stdoutf=outp.append, stderrf=None, log=False) ## # If outp contains some output it means modifications have been made if outp: raise CheckoutModifiedError( 'There are modifications to %s, please commit them or revert them before continuing' % dstDir) runSystemEx('rm -rf ' + dstDir, log=True) runSystemEx(' '.join(cmd), log=True)
def checkout(self, options, repo, repoPath, outputPath, branch): fullPath = os.path.join(repo.repoUrl, branch, repoPath) stderr = [] try: # Test to see if it's already checked out, if not continue self._raiseIfCheckout(outputPath) commands.runSingleProgramEx('svn co %s %s' % (fullPath, outputPath), stdoutf=None, stderrf=stderr.append, log=logging.DEBUG) except CheckoutModifiedError: logging.errorPrint('You have uncommited changes to %s, please revert changes or commit them' % outputPath) raise except commands.ProgramRunError: if 'refers to a file, not a directory' in ''.join(stderr): try: tmpPath = os.path.dirname(os.path.join(options('general.codir'), repoPath)) self._raiseIfCheckout(tmpPath) commands.runSystem('rm -rf ' + tmpPath, log=logging.DEBUG) commands.runSystemEx('mkdir -p ' + tmpPath, log=logging.DEBUG) commands.runSingleProgramEx('svn co %s %s' % (os.path.dirname(fullPath), tmpPath), stdoutf=None, stderrf=logging.errorPrintS, log=logging.DEBUG) commands.runSystem('rm -rf ' + outputPath, log=logging.DEBUG) commands.runSystemEx('ln -s %s %s' % (os.path.join(options('general.codir'), repoPath), outputPath), log=logging.DEBUG) except CheckoutModifiedError: logging.errorPrint('You have uncommited changes to %s, please revert changes or commit them' % tmpPath) raise else: for l in stderr: logging.errorPrintS(l) raise logExport(options('general.config_dir'), repo, repoPath, outputPath, branch, CHECKOUT)
def createMasterDataFile(cluster, machineConf): """ Creates a master data file as the perl start_cluster works """ template = open(cluster.config('cluster.master_user_data_tmpl')).read() clusterPrivateKey = open(cluster.config('cluster.cluster_private_key')).read() outf = [] runSingleProgramEx('ssh-keygen -y -f ' + cluster.config('cluster.cluster_private_key'), outf.append, None, log=logging.DEBUG) clusterPublicKey = ''.join(outf) template = template.replace('<TMPL_VAR NAME=CLUSTER_PRIVATE_KEY>', clusterPrivateKey) template = template.replace('<TMPL_VAR NAME=CLUSTER_PUBLIC_KEY>', clusterPublicKey) # Need to escape the ${ for bash template = template.replace('<TMPL_VAR NAME=MACHINE_CONF>', open(machineConf).read().replace('${', '\\${')) outf = os.path.join(cluster.config('general.secure_tmp'), 'master_user_data.%s.sh' % global_state.make_ref()) open(outf, 'w').write(template) return outf
def runWithCred(cred, cmd, stdoutf=logging.OUTSTREAM.write, stderrf=logging.ERRSTREAM.write, log=False): cmdPrefix = '' if hasattr(cred, 'ec2Path'): cmdPrefix = cred.ec2Path + '/' return commands.runSingleProgramEx(cmdPrefix + ' '.join(addCredInfo(cmd, cred)), stdoutf, stderrf, log=log, addEnv=cred.env)
def calculateMD5(files): stdout = [] commands.runSingleProgramEx('cat %s | md5sum' % ' '.join(files), stdoutf=stdout.append, stderrf=None, log=True) return stdout[-1].split(' ', 1)[0]
def writeKeyData(authKeysPath, keyData, user): fout = open(authKeysPath, 'a') fout.write('\n' + keyData + '\n') fout.close() runSingleProgramEx('chown %s:%s %s' % (user, user,authKeysPath), None, None) runSingleProgramEx('chmod 600 ' + authKeysPath, None, None)
def untarToDir(tarFile, outDir): commands.runSingleProgramEx('cd %s; tar -zxvf %s' % (outDir, tarFile), stdoutf=None, stderrf=None, log=logging.DEBUG)
def main(options, _args): logging.DEBUG = options('debug') env = {'PATH': os.path.join(options('prefix'), 'bin') + ':' + os.getenv('PATH')} tmpDir = os.path.join(options('tmpdir'), 'ca-ocaml-platform') buildDir = os.path.join(tmpDir, 'build') commands.runSystem('rm -rf ' + tmpDir, log=logging.DEBUG) commands.runSystemEx('mkdir -p ' + tmpDir, log=logging.DEBUG) commands.runSystemEx('mkdir -p ' + buildDir, log=logging.DEBUG) # # Download everything first for _, url in URLS: commands.runSystemEx('wget --quiet -P %s %s' % (tmpDir, url), log=logging.DEBUG) for project, url in URLS: fileName = os.path.basename(url) downloadedName = os.path.join(tmpDir, fileName) untarToDir(downloadedName, buildDir) # Ocaml gets special attention because it needs the prefix if project == 'ocaml': commands.runSingleProgramEx('cd %s/*%s*; ./configure -prefix %s' % (buildDir, project, options('prefix')), stdoutf=None, stderrf=sys.stderr.write, log=logging.DEBUG) commands.runSingleProgramEx('cd %s/*%s*; make world.opt install' % (buildDir, project), stdoutf=None, stderrf=sys.stderr.write, log=logging.DEBUG) elif project in ['findlib']: commands.runSingleProgramEx('cd %s/*%s*; ./configure' % (buildDir, project), stdoutf=None, stderrf=sys.stderr.write, addEnv=env, log=logging.DEBUG) commands.runSingleProgramEx('cd %s/*%s*; make all opt install' % (buildDir, project), stdoutf=None, stderrf=sys.stderr.write, addEnv=env, log=logging.DEBUG) elif project == 'core': commands.runSingleProgramEx('cd %s/*%s*; patch -p1 -i %s/patches/core-0.6.0-3.12.0.patch' % (buildDir, project, os.getcwd()), stdoutf=None, stderrf=sys.stderr.write, log=logging.DEBUG) commands.runSingleProgramEx('cd %s/*%s*; make' % (buildDir, project), stdoutf=None, stderrf=sys.stderr.write, addEnv=env, log=logging.DEBUG) commands.runSingleProgramEx('cd %s/*%s*; make install' % (buildDir, project), stdoutf=None, stderrf=sys.stderr.write, addEnv=env, log=logging.DEBUG) else: commands.runSingleProgramEx('cd %s/*%s*; make' % (buildDir, project), stdoutf=None, stderrf=sys.stderr.write, addEnv=env, log=logging.DEBUG) commands.runSingleProgramEx('cd %s/*%s*; make install' % (buildDir, project), stdoutf=None, stderrf=sys.stderr.write, addEnv=env, log=logging.DEBUG) commands.runSystemEx('rm -rf %s/*' % buildDir, log=logging.DEBUG) print 'Be sure to update your environment variables:' print 'export PATH=%s:$PATH' % (os.path.join(options('prefix'), 'bin'),)
def _raiseIfCheckout(self, path): outp = [] commands.runSingleProgramEx('svn status ' + path, stdoutf=outp.append, stderrf=None, log=False) # If outp contains some output it means modifications have been made if outp: raise CheckoutModifiedError('There are modifications to %s, please commit them or revert them before continuing' % path)
def ungzFile(fname): runSingleProgramEx('gzip -dc %s > %s' % (fname, fname[:-3])) return fname[:-3]
def bunzip2File(fname): stdout = [] runSingleProgramEx('bzcat %s | tar -C %s -xv' % (fname, os.path.dirname(fname)), stdout.append, None) return (os.path.join(os.path.dirname(fname), i.strip()) for i in stdout)
def untargzFile(fname): stdout = [] runSingleProgramEx('tar -C %s -zxvf %s' % (os.path.dirname(fname), fname), stdout.append, None) return (os.path.join(os.path.dirname(fname), i.strip()) for i in stdout)
def bunzip2File(fname): stdout = [] runSingleProgramEx( 'bzcat %s | tar -C %s -xv' % (fname, os.path.dirname(fname)), stdout.append, None) return (os.path.join(os.path.dirname(fname), i.strip()) for i in stdout)
def getSizeOfFiles(files): stdoutL = [] commands.runSingleProgramEx("du -kcs " + " ".join(files), stdoutf=stdoutL.append, stderrf=None, log=False) return int(stdoutL[-1].split()[0])
def networkingEnabled(): res = [] runSingleProgramEx('/sbin/ifconfig', res.append, None) return [l for l in res if 'inet addr:' in l and '127.0.0.1' not in l]
def calculateMD5(files): stdout = [] commands.runSingleProgramEx("cat %s | md5sum" % " ".join(files), stdoutf=stdout.append, stderrf=None, log=True) return stdout[-1].split(" ", 1)[0]