def downloadPipeline(instance, conf, pipelineId, outDir, outBaseName, overwrite=False, log=False):
"""
Downloads a pipeline from an instance.
conf - contains various config data, the following options need to be present:
ssh.options - options to pass to ssh
ssh.user - user to ssh as
dirs.clovr_project - the directory on the imave clovr project is expected to live
pipelineId - the id of the pipeline to download
outDir - local directory to download to
outBaseName - The basename for the output file
overwrite - if the downloaded file exists locally already, should it be downloaded (default No)
"""
outF = '/mnt/%s.tar.gz' % outBaseName
cmd = ['cd %s;' % conf('dirs.clovr_project'),
'tar',
'-zcf',
outF,
'output_repository/*/%s_*' % pipelineId,
'workflow/runtime/*/%s_*' % pipelineId]
runSystemInstanceEx(instance, ' '.join(cmd), None, (log and errorPrintS or None), user=conf('ssh.user'), options=conf('ssh.options'), log=log)
outFilename = os.path.join(outDir, os.path.basename(outF))
fileExists = os.path.exists(outFilename)
if fileExists and overwrite or not fileExists:
runSystemEx('mkdir -p ' + outDir)
scpFromEx(instance['public_dns'], outF, outDir, user=conf('ssh.user'), options=conf('ssh.options'), log=log)
runSystemInstanceEx(instance, 'rm ' + outF, None, (log and errorPrintS or None), user=conf('ssh.user'), options=conf('ssh.options'), log=log)
return outFilename
else:
runSystemInstanceEx(instance, 'rm ' + outF, None, (log and errorPrintS or None), user=conf('ssh.user'), options=conf('ssh.options'), log=log)
raise DownloadPipelineOverwriteError(outFilename)
def main(options, _args):
if options('general.devel'):
if not os.path.exists('/mnt/keys/devel1.pem'):
runSystemEx('cp %s /mnt/keys' % options('general.devel'))
else:
if not os.path.exists('/mnt/keys/devel1.pem'):
runSingleProgramEx('ssh-keygen -f /mnt/keys/devel1.pem -P ""',
None, None)
keyData = []
runSingleProgramEx('ssh-keygen -y -f /mnt/keys/devel1.pem', keyData.append,
sys.stderr.write)
keyData = ''.join(keyData)
for path, user in [('/root', 'root'), ('/home/www-data', 'www-data')]:
authorizedKeysPath = os.path.join(path, '.ssh', 'authorized_keys')
runSingleProgramEx('mkdir -p %s' % os.path.dirname(authorizedKeysPath),
None, None)
if os.path.exists(authorizedKeysPath):
authorizedKeys = open(authorizedKeysPath).read()
if keyData not in authorizedKeys:
writeKeyData(authorizedKeysPath, keyData, user)
else:
writeKeyData(authorizedKeysPath, keyData, user)
print
print
print 'Setup complete.'
print '*** Remember, currently you have to do this every time you restart the VM'
def main(options, _args):
if options('general.devel'):
if not os.path.exists('/mnt/keys/devel1.pem'):
runSystemEx('cp %s /mnt/keys' % options('general.devel'))
else:
if not os.path.exists('/mnt/keys/devel1.pem'):
runSingleProgramEx('ssh-keygen -f /mnt/keys/devel1.pem -P ""', None, None)
keyData = []
runSingleProgramEx('ssh-keygen -y -f /mnt/keys/devel1.pem', keyData.append, sys.stderr.write)
keyData = ''.join(keyData)
for path, user in [('/root', 'root'), ('/home/www-data', 'www-data')]:
authorizedKeysPath = os.path.join(path, '.ssh', 'authorized_keys')
runSingleProgramEx('mkdir -p %s' % os.path.dirname(authorizedKeysPath), None, None)
if os.path.exists(authorizedKeysPath):
authorizedKeys = open(authorizedKeysPath).read()
if keyData not in authorizedKeys:
writeKeyData(authorizedKeysPath, keyData, user)
else:
writeKeyData(authorizedKeysPath, keyData, user)
print
print
print 'Setup complete.'
print '*** Remember, currently you have to do this every time you restart the VM'
def addJavaCert(outputdir, host, port):
"""This installs a java cert. It is assumed that install-cert.sh is in the PATH"""
runSystemEx("""cd /tmp/ && echo 1 | install-cert.sh %s:%d""" %
(host, port))
if outputdir not in ['/tmp', '/tmp/']:
runSystem("""mv /tmp/jssecacerts %s""" % (outputdir, ))
def bundleAMI(chan):
options, rchan = chan.receive()
try:
cmd = ['ec2-bundle-image',
'-c ${general.cert}',
'-k ${general.key}',
'-u ${general.user}',
'--kernel ${general.kernel}',
'-i ${general.image}',
'-d ${general.dest}',
'-p ${general.image}',
'-r ${general.arch}']
if options('general.ec2cert'):
cmd.append('--ec2cert ${general.ec2cert}')
runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug'))
cmd = ['ec2-upload-bundle', '-b ${general.image}', '-m ${general.dest}/${general.image}.manifest.xml', '-a ${general.access_key}', '-s ${general.secret_access_key}']
runSystemEx(config.replaceStr(' '.join(cmd), options), log=options('general.debug'))
cmd = ['ec2-register', '${general.image}/${general.image}.manifest.xml', '-K ${general.key}', '-C ${general.cert}']
outp = []
runSingleProgramEx(config.replaceStr(' '.join(cmd), options), stdoutf=outp.append, stderrf=sys.stderr.write, log=True)
ami = ''.join(outp).split()[-1]
##
# Make the AMI public
runSingleProgramEx('ec2-modify-image-attribute %s --launch-permission -a all' % ami, stdoutf=sys.stdout.write, stderrf=sys.stderr.write, log=True)
rchan.send(ami)
except Exception, err:
rchan.sendError(err)
def instantiateCredential(conf, cred):
if not conf('config_loaded', default=False):
conf = config.configFromMap({'config_loaded': True},
base=config.configFromStream(open(
conf('general.conf_file')),
base=conf))
certFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_cert.pem')
keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem')
if not os.path.exists(certFile) and not os.path.exists(keyFile):
tmpCertFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_cert-tmp.pem')
tmpKeyFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_key-tmp.pem')
if 'ec2_url' not in cred.metadata:
raise Exception('You must have an ec2_url')
parsedUrl = urlparse.urlparse(cred.metadata['ec2_url'])
if ':' not in parsedUrl.netloc:
raise Exception('Your URL must contain a port')
host, port = parsedUrl.netloc.split(':')
fout = open(tmpCertFile, 'w')
fout.write(cred.cert)
fout.close()
fout = open(tmpKeyFile, 'w')
fout.write(cred.pkey)
fout.close()
try:
commands.runSystemEx(' '.join([
'nimbusCerts2EC2.py', '--in-cert=' + tmpCertFile,
'--out-cert=' + certFile, '--in-key=' + tmpKeyFile,
'--out-key=' + keyFile, '--java-cert-dir=/tmp',
'--java-cert-host=' + host, '--java-cert-port=' + port
]) + ' > /dev/null 2>&1',
log=True)
commands.runSystemEx('chmod +r ' + keyFile)
finally:
os.unlink(tmpCertFile)
os.unlink(tmpKeyFile)
ec2Home = '/opt/ec2-api-tools-1.3-42584'
newCred = func.Record(
cert=certFile,
pkey=keyFile,
ec2Path=os.path.join(ec2Home, 'bin'),
env=dict(EC2_JVM_ARGS='-Djavax.net.ssl.trustStore=/tmp/jssecacerts',
EC2_HOME=ec2Home,
EC2_URL=cred.metadata['ec2_url']))
if os.path.exists(conf('cluster.cluster_private_key') + '.pub'):
pubKey = open(conf('cluster.cluster_private_key') +
'.pub').read().rstrip()
ec2_control.addKeypair(newCred,
'"' + conf('cluster.key') + '||' + pubKey + '"')
return (conf, newCred)
def sharedFoldersEnabled():
try:
##
# Are we running VMWare?
runSystemEx('vmware-checkvm > /dev/null 2>&1')
res = []
runSingleProgramEx('df', res.append, None)
return [l for l in res if l.startswith('.host:/shared')]
except:
##
# If we aren't running VMWare, just assume it worked for now
return True
def sharedFoldersEnabled():
try:
##
# Are we running VMWare?
runSystemEx('vmware-checkvm > /dev/null 2>&1')
res = []
runSingleProgramEx('df', res.append, None)
return [l for l in res if l.startswith('.host:/shared')]
except:
##
# If we aren't running VMWare, just assume it worked for now
return True
def convertKey(inname, outname):
"""
This just calls an openssl command.
The user will have to type in their password
"""
##
# Using os.system here because we want to let the user type in
cmd = """openssl rsa -in %s -out %s""" % (inname, outname)
code = os.system(cmd)
if code != 0:
raise ProgramRunError(cmd, code)
runSystemEx('chmod 400 ' + outname)
def body(self):
form = cgi.FieldStorage()
host = form['host'].value
sio = StringIO.StringIO()
sgeRootBin = os.path.join(SGE_ROOTBIN)
if 'ipaddr' in form and re.match('^\d+\.\d+\.\d+\.\d+$', form['ipaddr'].value):
commands.runSystemEx('addEtcHosts.py %s %s' % (host, form['ipaddr'].value))
cmd = [os.path.join(sgeRootBin, 'qconf'),
'-ah',
host,
'&>',
'/tmp/add_host.out']
commands.runSystemEx(' '.join(cmd))
def convertKey(inname, outname):
"""
This just calls an openssl command.
The user will have to type in their password
"""
##
# Using os.system here because we want to let the user type in
cmd = """openssl rsa -in %s -out %s""" % (inname, outname)
code = os.system(cmd)
if code != 0:
raise ProgramRunError(cmd, code)
runSystemEx('chmod 400 ' + outname)
def checkout(self, options, repo, repoPath, outputPath, branch):
fullPath = os.path.join(repo.repoUrl, branch, repoPath)
stderr = []
try:
# Test to see if it's already checked out, if not continue
self._raiseIfCheckout(outputPath)
commands.runSingleProgramEx('svn co %s %s' %
(fullPath, outputPath),
stdoutf=None,
stderrf=stderr.append,
log=logging.DEBUG)
except CheckoutModifiedError:
logging.errorPrint(
'You have uncommited changes to %s, please revert changes or commit them'
% outputPath)
raise
except commands.ProgramRunError:
if 'refers to a file, not a directory' in ''.join(stderr):
try:
tmpPath = os.path.dirname(
os.path.join(options('general.codir'), repoPath))
self._raiseIfCheckout(tmpPath)
commands.runSystem('rm -rf ' + tmpPath, log=logging.DEBUG)
commands.runSystemEx('mkdir -p ' + tmpPath,
log=logging.DEBUG)
commands.runSingleProgramEx(
'svn co %s %s' % (os.path.dirname(fullPath), tmpPath),
stdoutf=None,
stderrf=logging.errorPrintS,
log=logging.DEBUG)
commands.runSystem('rm -rf ' + outputPath,
log=logging.DEBUG)
commands.runSystemEx('ln -s %s %s' % (os.path.join(
options('general.codir'), repoPath), outputPath),
log=logging.DEBUG)
except CheckoutModifiedError:
logging.errorPrint(
'You have uncommited changes to %s, please revert changes or commit them'
% tmpPath)
raise
else:
for l in stderr:
logging.errorPrintS(l)
raise
logExport(options('general.config_dir'), repo, repoPath, outputPath,
branch, CHECKOUT)
def startUpDevNode(conf):
"""
DEPRECATED: This type is deprecated, instead we should just be using the [dev] section of
the config file to specify what branches and what parts of the system to load from
svn
Steps in starting a dev node:
1 - Remove /usr/local/stow (this should be a config option eventually)
2 - Check out /usr/local/stow
3 - Check out /opt/packages
Any SVN work is done on trunk (need to add config to specify a branch)
"""
executePolicyDirWEx('/opt/config_policies', 'DEV')
runSystemEx("""updateAllDirs.py --co""")
def startUpDevNode(conf):
"""
DEPRECATED: This type is deprecated, instead we should just be using the [dev] section of
the config file to specify what branches and what parts of the system to load from
svn
Steps in starting a dev node:
1 - Remove /usr/local/stow (this should be a config option eventually)
2 - Check out /usr/local/stow
3 - Check out /opt/packages
Any SVN work is done on trunk (need to add config to specify a branch)
"""
executePolicyDirWEx('/opt/config_policies', 'DEV')
runSystemEx("""updateAllDirs.py --co""")
def body(self):
form = cgi.FieldStorage()
host = form['host'].value
sio = StringIO.StringIO()
sgeRootBin = os.path.join(SGE_ROOTBIN)
if 'ipaddr' in form and re.match('^\d+\.\d+\.\d+\.\d+$',
form['ipaddr'].value):
commands.runSystemEx('addEtcHosts.py %s %s' %
(host, form['ipaddr'].value))
cmd = [
os.path.join(sgeRootBin, 'qconf'), '-ah', host, '&>',
'/tmp/add_host.out'
]
commands.runSystemEx(' '.join(cmd))
def bundleAMI(chan):
options, rchan = chan.receive()
try:
cmd = [
'ec2-bundle-image', '-c ${general.cert}', '-k ${general.key}',
'-u ${general.user}', '--kernel ${general.kernel}',
'-i ${general.image}', '-d ${general.dest}', '-p ${general.image}',
'-r ${general.arch}'
]
if options('general.ec2cert'):
cmd.append('--ec2cert ${general.ec2cert}')
runSystemEx(config.replaceStr(' '.join(cmd), options),
log=options('general.debug'))
cmd = [
'ec2-upload-bundle', '-b ${general.image}',
'-m ${general.dest}/${general.image}.manifest.xml',
'-a ${general.access_key}', '-s ${general.secret_access_key}'
]
runSystemEx(config.replaceStr(' '.join(cmd), options),
log=options('general.debug'))
cmd = [
'ec2-register', '${general.image}/${general.image}.manifest.xml',
'-K ${general.key}', '-C ${general.cert}'
]
outp = []
runSingleProgramEx(config.replaceStr(' '.join(cmd), options),
stdoutf=outp.append,
stderrf=sys.stderr.write,
log=True)
ami = ''.join(outp).split()[-1]
##
# Make the AMI public
runSingleProgramEx(
'ec2-modify-image-attribute %s --launch-permission -a all' % ami,
stdoutf=sys.stdout.write,
stderrf=sys.stderr.write,
log=True)
rchan.send(ami)
except Exception, err:
rchan.sendError(err)
def runTaskStatus(taskName, clusterName=None, hostName='localhost'):
"""
This is a simple function that takes a taskname and simply runs vp-describe-task
on it. It is meant to be used in specific situations in front ends. It is not
meant to be a generic function. There are no guarantees that this funciton will
exist tomorrow and it could be moved into a more fitting location at any point
"""
cmd = [
'vp-describe-task', '--show', '--show-error', '--exit-code', '--block',
'--no-print-polling'
]
if clusterName:
cmd.append('--name=' + clusterName)
cmd.append('--host=' + hostName)
cmd.append(taskName)
commands.runSystemEx(' '.join(cmd))
def instantiateCredential(conf, cred):
if not conf('config_loaded', default=False):
conf = config.configFromMap({'config_loaded': True},
base=config.configFromStream(open(conf('general.conf_file')), base=conf))
certFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert.pem')
keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem')
if not os.path.exists(certFile) and not os.path.exists(keyFile):
tmpCertFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert-tmp.pem')
tmpKeyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key-tmp.pem')
if 'ec2_url' not in cred.metadata:
raise Exception('You must have an ec2_url')
parsedUrl = urlparse.urlparse(cred.metadata['ec2_url'])
if ':' not in parsedUrl.netloc:
raise Exception('Your URL must contain a port')
host, port = parsedUrl.netloc.split(':')
fout = open(tmpCertFile, 'w')
fout.write(cred.cert)
fout.close()
fout = open(tmpKeyFile, 'w')
fout.write(cred.pkey)
fout.close()
try:
commands.runSystemEx(' '.join(['nimbusCerts2EC2.py',
'--in-cert=' + tmpCertFile,
'--out-cert=' + certFile,
'--in-key=' + tmpKeyFile,
'--out-key=' + keyFile,
'--java-cert-dir=/tmp',
'--java-cert-host=' + host,
'--java-cert-port=' + port]) + ' > /dev/null 2>&1', log=True)
commands.runSystemEx('chmod +r ' + keyFile)
finally:
os.unlink(tmpCertFile)
os.unlink(tmpKeyFile)
ec2Home = '/opt/ec2-api-tools-1.3-42584'
newCred = func.Record(cert=certFile, pkey=keyFile, ec2Path=os.path.join(ec2Home, 'bin'),
env=dict(EC2_JVM_ARGS='-Djavax.net.ssl.trustStore=/tmp/jssecacerts',
EC2_HOME=ec2Home,
EC2_URL=cred.metadata['ec2_url']))
if os.path.exists(conf('cluster.cluster_private_key') + '.pub'):
pubKey = open(conf('cluster.cluster_private_key') + '.pub').read().rstrip()
ec2_control.addKeypair(newCred, '"' + conf('cluster.key') + '||' + pubKey + '"')
return (conf, newCred)
def runTaskStatus(taskName, clusterName=None, hostName='localhost'):
"""
This is a simple function that takes a taskname and simply runs vp-describe-task
on it. It is meant to be used in specific situations in front ends. It is not
meant to be a generic function. There are no guarantees that this funciton will
exist tomorrow and it could be moved into a more fitting location at any point
"""
cmd = ['vp-describe-task',
'--show',
'--show-error',
'--exit-code',
'--block',
'--no-print-polling']
if clusterName:
cmd.append('--name=' + clusterName)
cmd.append('--host=' + hostName)
cmd.append(taskName)
commands.runSystemEx(' '.join(cmd))
def startUpAllNodes(conf):
"""
Things that need to be done for all nodes:
1 - Go through /usr/local/stow and /opt/opt-packages installing all packages
2 - List through all .py files in /opt/config_policies and run them (eventually this directory should probably
be split up by node type)
"""
if conf('dev.update_dirs', default=None):
cmd = [
'updateAllDirs.py',
'--clovr-branch=' + conf('dev.clovr_branch', default=''),
'--vappio-branch=' + conf('dev.vappio_branch', default=''),
conf('dev.update_dirs')
]
runSystemEx(' '.join(cmd))
installAllStow()
installAllOptPackages()
executePolicyDirWEx(startPolicy, '/opt/config_policies')
def main(options, args):
logging.DEBUG = options('general.debug')
urls = []
if not args:
for line in [l for l in sys.stdin if l.strip()]:
urls.append(line.strip() + options('general.suffix'))
else:
urls.extend([u + options('general.suffix') for u in args])
for url in urls:
reliableDownloader.deleteDownloadedFiles('/tmp', url)
commands.runSystemEx(' '.join(['wget', '-P', '/tmp', '-nv', url]),
log=logging.DEBUG)
for f in reliableDownloader.getDownloadFilenames('/tmp', url):
sys.stdout.write('\n'.join([
'%s %s' % (l.strip().split()[0],
os.path.join(os.path.dirname(url),
l.strip().split()[1])) for l in open(f)
]) + '\n')
reliableDownloader.deleteDownloadedFiles('/tmp', url)
def startUpAllNodes(conf):
"""
Things that need to be done for all nodes:
1 - Go through /usr/local/stow and /opt/opt-packages installing all packages
2 - List through all .py files in /opt/config_policies and run them (eventually this directory should probably
be split up by node type)
"""
if conf('dev.update_dirs', default=None):
cmd = ['updateAllDirs.py',
'--clovr-branch=' + conf('dev.clovr_branch', default=''),
'--vappio-branch=' + conf('dev.vappio_branch', default=''),
conf('dev.update_dirs')]
runSystemEx(' '.join(cmd))
installAllStow()
installAllOptPackages()
executePolicyDirWEx(startPolicy, '/opt/config_policies')
def grabFromSVN(options, srcUrl, branch, d, dstDir):
cmd = ['echo p | svn']
if options('general.co'):
cmd += ['co']
else:
cmd += ['export', '--force']
cmd += [srcUrl + '/' + branch + '/' + d, dstDir]
outp = []
runSingleProgramEx('svn status ' + dstDir,
stdoutf=outp.append,
stderrf=None,
log=False)
##
# If outp contains some output it means modifications have been made
if outp:
raise CheckoutModifiedError(
'There are modifications to %s, please commit them or revert them before continuing'
% dstDir)
runSystemEx('rm -rf ' + dstDir, log=True)
runSystemEx(' '.join(cmd), log=True)
def checkout(self, options, repo, repoPath, outputPath, branch):
fullPath = os.path.join(repo.repoUrl, branch, repoPath)
stderr = []
try:
# Test to see if it's already checked out, if not continue
self._raiseIfCheckout(outputPath)
commands.runSingleProgramEx('svn co %s %s' % (fullPath, outputPath),
stdoutf=None,
stderrf=stderr.append,
log=logging.DEBUG)
except CheckoutModifiedError:
logging.errorPrint('You have uncommited changes to %s, please revert changes or commit them' % outputPath)
raise
except commands.ProgramRunError:
if 'refers to a file, not a directory' in ''.join(stderr):
try:
tmpPath = os.path.dirname(os.path.join(options('general.codir'), repoPath))
self._raiseIfCheckout(tmpPath)
commands.runSystem('rm -rf ' + tmpPath, log=logging.DEBUG)
commands.runSystemEx('mkdir -p ' + tmpPath, log=logging.DEBUG)
commands.runSingleProgramEx('svn co %s %s' % (os.path.dirname(fullPath), tmpPath),
stdoutf=None,
stderrf=logging.errorPrintS,
log=logging.DEBUG)
commands.runSystem('rm -rf ' + outputPath, log=logging.DEBUG)
commands.runSystemEx('ln -s %s %s' % (os.path.join(options('general.codir'), repoPath),
outputPath),
log=logging.DEBUG)
except CheckoutModifiedError:
logging.errorPrint('You have uncommited changes to %s, please revert changes or commit them' % tmpPath)
raise
else:
for l in stderr:
logging.errorPrintS(l)
raise
logExport(options('general.config_dir'), repo, repoPath, outputPath, branch, CHECKOUT)
def main(options, _args):
cmd = ['ec2-bundle-image',
'-c ${general.cert}',
'-k ${general.key}',
'-u ${general.user}',
'--kernel ${general.kernel}',
'-i ${general.image}',
'-d ${general.dest}',
'-p ${general.image}',
'-r ${general.arch}']
if options('general.ec2cert'):
cmd.append('--ec2cert ${general.ec2cert}')
runSystemEx(replaceStr(' '.join(cmd), options), log=options('general.debug'))
cmd = ['ec2-upload-bundle', '-b ${general.image}', '-m ${general.dest}/${general.image}.manifest.xml', '-a ${general.access_key}', '-s ${general.secret_access_key}']
runSystemEx(replaceStr(' '.join(cmd), options), log=options('general.debug'))
cmd = ['ec2-register', '${general.image}/${general.image}.manifest.xml', '-K ${general.key}', '-C ${general.cert}']
##
# We want to output the AMI regardless
runSystemEx(replaceStr(' '.join(cmd), options), log=True)
def main(options, _args):
open('/opt/CLOVR-RELEASE', 'w').write(options('general.version_string') + '\n')
for c in COMMANDS:
runSystemEx(c, log=True)
def main(options, _args):
runSystemEx('svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')),
log=True)
runSystemEx('svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')),
log=True)
runSystemEx('scp %s:/export/%s .' % (options('general.remote_name'), options('general.image')), log=True)
runSystemEx('cp %s /usr/local/projects/clovr/images' % options('general.image'), log=True)
runSystemEx('cp %s VMware_conversion/shared/convert_img.img' % options('general.image'), log=True)
convertChannel = threads.runThreadWithChannel(convertImage).channel.sendWithChannel(options)
waitForPasswordChange()
bundleChannel = threads.runThreadWithChannel(bundleAMI).channel.sendWithChannel(options)
try:
convertChannel.receive()
vmWareDir = 'clovr-vmware.%s' % options('general.version')
runSystemEx('mkdir -p ' + vmWareDir, log=True)
runSystemEx('mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join(vmWareDir, 'clovr.9-04.x86-64.%s.vmdk' % options('general.version')))
runSystemEx('mkdir -p %s %s' % (os.path.join(vmWareDir, 'keys'),
os.path.join(vmWareDir, 'user_data')), log=True)
runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir, log=True)
fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w')
clovrConf = config.configFromMap(dict(version=options('general.version')))
for line in open('/usr/local/projects/clovr/start_clovr.vmx'):
fout.write(config.replaceStr(line, clovrConf))
except Exception, err:
errorPrint('Converting image failed. Error message:')
errorPrint(str(err))
def runCommand(_ctype, _baseDir, command, _tagfile):
runSystemEx(command)
def tagData(tagsDir,
tagName,
tagBaseDir,
files,
recursive,
expand,
compress,
append,
overwrite,
metadata=None,
filterF=None):
"""
Tag a list of files with the name. The files can contain direcotires, and if recursive
is set the contends of the directories will become part of the tag rather than just the name
tagBaseDir is the name of the directory that is not part of the actual tag heirarchy
expand will cause any archives listed to be expanded and the contents of the archive to be added
compress will compress the files that have been put in the tag. compress should be the path to the
directory the compressed file should be put.
append will add to a tagName if it already exists, only unique names will be kept though
filterF - if you want to filter any of the files as they are added to the file list provide a filter
function that will be called on each individual file name. The file will be added if filter returns True
This returns the tag that was created
"""
if metadata is None:
metadata = {}
if not os.path.exists(tagsDir):
runSystemEx('mkdir -p ' + tagsDir)
outName = os.path.join(tagsDir, tagName)
if os.path.exists(outName) and not append and not overwrite:
raise Exception('Tag already exists')
##
# Keep a set of all old entries in the file, when we walk the generator we'll
# we'll check to see if the file already exists in here
if append and os.path.exists(outName):
oldFiles = set([l.strip() for l in open(outName)])
else:
oldFiles = set()
files = [
f for f in generateFileList(files, recursive, expand)
if f not in oldFiles and (not filterF or filterF and filterF(f))
]
if overwrite:
##
# If we are just overwritign the file, no need to old the list of oldFiles
# Technically it shouldn't matter but if the old file list is really large
# the lookup could be expensive
outFile = open(outName, 'w')
oldFiles = set()
else:
outFile = open(outName, 'a')
outFile.write('\n'.join(files))
outFile.write('\n')
outFile.close()
#
# If we are compressing the files then, load the tag back up
# so we have all of the files there
if compress:
outTar = str(os.path.join(compress, tagName + '.tar'))
outGzip = outTar + '.gz'
if os.path.exists(outGzip):
os.remove(outGzip)
runSystemEx('mkdir -p ' + compress)
files = loadTagFile(outName)('files')
baseDirFiles, nonBaseDirFiles = partitionFiles(files, tagBaseDir)
if baseDirFiles:
for fs in func.chunk(20, baseDirFiles):
cmd = [
'tar',
'-C',
tagBaseDir,
'-rf',
outTar,
]
cmd.extend([removeBase('/', f) for f in fs])
runSystemEx(' '.join(cmd), log=True)
if nonBaseDirFiles:
for fs in func.chunk(20, nonBaseDirFiles):
cmd = [
'tar',
'-C',
'/',
'-rf',
outTar,
]
cmd.extend([removeBase('/', f) for f in fs])
runSystemEx(' '.join(cmd), log=True)
#
# It's possible we have no values here, if so, the tar was not created
# and should be ignored
if os.path.exists(outTar):
runSystemEx('gzip ' + outTar, log=True)
metadata = func.updateDict(metadata, {
'compressed': True,
'compressed_file': outGzip
})
#
# If tagBaseDir is set it means we have some metadata to write
if tagBaseDir:
metadata['tag_base_dir'] = tagBaseDir
if append and os.path.exists(outName + '.metadata'):
tmd = json.loads(open(outName + '.metadata').read())
metadata = func.updateDict(tmd, metadata)
outFile = open(outName + '.metadata', 'w')
outFile.write(json.dumps(metadata, indent=1) + '\n')
outFile.close()
return loadTagFile(outName)
def main(options, _args):
logging.DEBUG = options('debug')
env = {'PATH': os.path.join(options('prefix'), 'bin') + ':' + os.getenv('PATH')}
tmpDir = os.path.join(options('tmpdir'), 'ca-ocaml-platform')
buildDir = os.path.join(tmpDir, 'build')
commands.runSystem('rm -rf ' + tmpDir, log=logging.DEBUG)
commands.runSystemEx('mkdir -p ' + tmpDir, log=logging.DEBUG)
commands.runSystemEx('mkdir -p ' + buildDir, log=logging.DEBUG)
#
# Download everything first
for _, url in URLS:
commands.runSystemEx('wget --quiet -P %s %s' % (tmpDir, url), log=logging.DEBUG)
for project, url in URLS:
fileName = os.path.basename(url)
downloadedName = os.path.join(tmpDir, fileName)
untarToDir(downloadedName, buildDir)
# Ocaml gets special attention because it needs the prefix
if project == 'ocaml':
commands.runSingleProgramEx('cd %s/*%s*; ./configure -prefix %s' % (buildDir, project, options('prefix')),
stdoutf=None,
stderrf=sys.stderr.write,
log=logging.DEBUG)
commands.runSingleProgramEx('cd %s/*%s*; make world.opt install' % (buildDir, project),
stdoutf=None,
stderrf=sys.stderr.write,
log=logging.DEBUG)
elif project in ['findlib']:
commands.runSingleProgramEx('cd %s/*%s*; ./configure' % (buildDir, project),
stdoutf=None,
stderrf=sys.stderr.write,
addEnv=env,
log=logging.DEBUG)
commands.runSingleProgramEx('cd %s/*%s*; make all opt install' % (buildDir, project),
stdoutf=None,
stderrf=sys.stderr.write,
addEnv=env,
log=logging.DEBUG)
elif project == 'core':
commands.runSingleProgramEx('cd %s/*%s*; patch -p1 -i %s/patches/core-0.6.0-3.12.0.patch' % (buildDir, project, os.getcwd()),
stdoutf=None,
stderrf=sys.stderr.write,
log=logging.DEBUG)
commands.runSingleProgramEx('cd %s/*%s*; make' % (buildDir, project),
stdoutf=None,
stderrf=sys.stderr.write,
addEnv=env,
log=logging.DEBUG)
commands.runSingleProgramEx('cd %s/*%s*; make install' % (buildDir, project),
stdoutf=None,
stderrf=sys.stderr.write,
addEnv=env,
log=logging.DEBUG)
else:
commands.runSingleProgramEx('cd %s/*%s*; make' % (buildDir, project),
stdoutf=None,
stderrf=sys.stderr.write,
addEnv=env,
log=logging.DEBUG)
commands.runSingleProgramEx('cd %s/*%s*; make install' % (buildDir, project),
stdoutf=None,
stderrf=sys.stderr.write,
addEnv=env,
log=logging.DEBUG)
commands.runSystemEx('rm -rf %s/*' % buildDir, log=logging.DEBUG)
print 'Be sure to update your environment variables:'
print 'export PATH=%s:$PATH' % (os.path.join(options('prefix'), 'bin'),)
def main(options, _args):
runSystemEx(
'svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"'
% (options('general.version'), options('general.version')),
log=True)
runSystemEx(
'svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"'
% (options('general.version'), options('general.version')),
log=True)
runSystemEx('scp %s:/export/%s .' %
(options('general.remote_name'), options('general.image')),
log=True)
runSystemEx('cp %s /usr/local/projects/clovr/images' %
options('general.image'),
log=True)
runSystemEx('cp %s VMware_conversion/shared/convert_img.img' %
options('general.image'),
log=True)
convertChannel = threads.runThreadWithChannel(
convertImage).channel.sendWithChannel(options)
waitForPasswordChange()
bundleChannel = threads.runThreadWithChannel(
bundleAMI).channel.sendWithChannel(options)
try:
convertChannel.receive()
vmWareDir = 'clovr-vmware.%s' % options('general.version')
runSystemEx('mkdir -p ' + vmWareDir, log=True)
runSystemEx(
'mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join(
vmWareDir,
'clovr.9-04.x86-64.%s.vmdk' % options('general.version')))
runSystemEx('mkdir -p %s %s' % (os.path.join(
vmWareDir, 'keys'), os.path.join(vmWareDir, 'user_data')),
log=True)
runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir,
log=True)
fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w')
clovrConf = config.configFromMap(
dict(version=options('general.version')))
for line in open('/usr/local/projects/clovr/start_clovr.vmx'):
fout.write(config.replaceStr(line, clovrConf))
except Exception, err:
errorPrint('Converting image failed. Error message:')
errorPrint(str(err))
def runSystemExC(conf, cmd):
cmd = replaceStr(cmd, conf)
runSystemEx(cmd)
def main(options, _args):
updateAll = False
for o in OPTIONS:
if o[0] not in ['vappio_branch', 'clovr_branch'
] and options('general.' + o[0]):
break
else:
updateAll = True
clovrBranch = options('general.clovr_branch')
vappioBranch = options('general.vappio_branch')
try:
if options('general.stow') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code',
clovrBranch, 'stow', '/usr/local/stow')
if options('general.opt_packages') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code',
clovrBranch, 'opt-packages', '/opt/opt-packages')
if options('general.config_policies') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code',
clovrBranch, 'config_policies', '/opt/config_policies')
if options('general.vappio_py') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code',
vappioBranch, 'vappio-py', '/opt/vappio-py')
runSystemEx("""chmod +x /opt/vappio-py/vappio/cli/*.py""")
if options('general.vappio_apps') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code',
vappioBranch, 'vappio-apps', '/opt/vappio-apps')
if options('general.vappio_twisted') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code',
vappioBranch, 'vappio-twisted', '/opt/vappio-twisted')
if options('general.vappio_scripts') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code',
vappioBranch, 'vappio-scripts', '/opt/vappio-scripts')
runSystemEx("""chmod -R +x /opt/vappio-scripts""", log=True)
runSystemEx("""cp -f /opt/vappio-scripts/clovrEnv.sh /root""",
log=True)
runSystemEx(
"""cp -f /opt/vappio-scripts/local /etc/init.d/local""",
log=True)
runSystemEx(
"""cp -f /opt/vappio-scripts/rc.local /etc/init.d/rc.local""",
log=True)
runSystemEx(
"""cp -f /opt/vappio-scripts/screenrc /root/.screenrc""",
log=True)
if options('general.clovr_pipelines') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code',
clovrBranch, 'clovr_pipelines', '/opt/clovr_pipelines')
if options('general.vappio_py_www') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code',
vappioBranch, 'vappio-www/py-www', '/var/www/vappio')
##
# Only want to do this one when specified
# if options('general.vappio_conf'):
# grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code', vappioBranch, 'vappio-conf', '/mnt/vappio-conf')
if options('general.hudson') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code',
clovrBranch, 'hudson/hudson-config/jobs',
'/var/lib/hudson/jobs')
grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code',
clovrBranch, 'hudson/hudson-scripts', '/opt/hudson')
runSystemEx("""chown -R hudson.nogroup /var/lib/hudson/jobs""",
log=True)
if options('general.clovr_www') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code',
clovrBranch, 'clovr-www', '/var/www/clovr')
except CheckoutModifiedError, err:
errorPrint(str(err))
def tagData(tagsDir, tagName, tagBaseDir, files, recursive, expand, compress, append, overwrite, metadata=None, filterF=None):
"""
Tag a list of files with the name. The files can contain direcotires, and if recursive
is set the contends of the directories will become part of the tag rather than just the name
tagBaseDir is the name of the directory that is not part of the actual tag heirarchy
expand will cause any archives listed to be expanded and the contents of the archive to be added
compress will compress the files that have been put in the tag. compress should be the path to the
directory the compressed file should be put.
append will add to a tagName if it already exists, only unique names will be kept though
filterF - if you want to filter any of the files as they are added to the file list provide a filter
function that will be called on each individual file name. The file will be added if filter returns True
This returns the tag that was created
"""
if metadata is None:
metadata = {}
if not os.path.exists(tagsDir):
runSystemEx('mkdir -p ' + tagsDir)
outName = os.path.join(tagsDir, tagName)
if os.path.exists(outName) and not append and not overwrite:
raise Exception('Tag already exists')
##
# Keep a set of all old entries in the file, when we walk the generator we'll
# we'll check to see if the file already exists in here
if append and os.path.exists(outName):
oldFiles = set([l.strip() for l in open(outName)])
else:
oldFiles = set()
files = [f
for f in generateFileList(files, recursive, expand)
if f not in oldFiles and (not filterF or filterF and filterF(f))]
if overwrite:
##
# If we are just overwritign the file, no need to old the list of oldFiles
# Technically it shouldn't matter but if the old file list is really large
# the lookup could be expensive
outFile = open(outName, 'w')
oldFiles = set()
else:
outFile = open(outName, 'a')
outFile.write('\n'.join(files))
outFile.write('\n')
outFile.close()
#
# If we are compressing the files then, load the tag back up
# so we have all of the files there
if compress:
outTar = str(os.path.join(compress, tagName + '.tar'))
outGzip = outTar + '.gz'
if os.path.exists(outGzip):
os.remove(outGzip)
runSystemEx('mkdir -p ' + compress)
files = loadTagFile(outName)('files')
baseDirFiles, nonBaseDirFiles = partitionFiles(files, tagBaseDir)
if baseDirFiles:
for fs in func.chunk(20, baseDirFiles):
cmd = ['tar',
'-C', tagBaseDir,
'-rf', outTar,
]
cmd.extend([removeBase('/', f) for f in fs])
runSystemEx(' '.join(cmd), log=True)
if nonBaseDirFiles:
for fs in func.chunk(20, nonBaseDirFiles):
cmd = ['tar',
'-C', '/',
'-rf', outTar,
]
cmd.extend([removeBase('/', f) for f in fs])
runSystemEx(' '.join(cmd), log=True)
#
# It's possible we have no values here, if so, the tar was not created
# and should be ignored
if os.path.exists(outTar):
runSystemEx('gzip ' + outTar, log=True)
metadata = func.updateDict(metadata, {'compressed': True,
'compressed_file': outGzip})
#
# If tagBaseDir is set it means we have some metadata to write
if tagBaseDir:
metadata['tag_base_dir'] = tagBaseDir
if append and os.path.exists(outName + '.metadata'):
tmd = json.loads(open(outName + '.metadata').read())
metadata = func.updateDict(tmd, metadata)
outFile = open(outName + '.metadata', 'w')
outFile.write(json.dumps(metadata, indent=1) + '\n')
outFile.close()
return loadTagFile(outName)
def runCommand(_ctype, _baseDir, command, _tagfile):
runSystemEx(command)
def addJavaCert(outputdir, host, port):
"""This installs a java cert. It is assumed that install-cert.sh is in the PATH"""
runSystemEx("""cd /tmp/ && echo 1 | install-cert.sh %s:%d""" % (host, port))
if outputdir not in ['/tmp', '/tmp/']:
runSystem("""mv /tmp/jssecacerts %s""" % (outputdir,))
def runSystemExC(conf, cmd):
cmd = replaceStr(cmd, conf)
runSystemEx(cmd)
