def main(options, _args):
runSystemEx('svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')),
log=True)
runSystemEx('svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')),
log=True)
runSystemEx('scp %s:/export/%s .' % (options('general.remote_name'), options('general.image')), log=True)
runSystemEx('cp %s /usr/local/projects/clovr/images' % options('general.image'), log=True)
runSystemEx('cp %s VMware_conversion/shared/convert_img.img' % options('general.image'), log=True)
convertChannel = threads.runThreadWithChannel(convertImage).channel.sendWithChannel(options)
waitForPasswordChange()
bundleChannel = threads.runThreadWithChannel(bundleAMI).channel.sendWithChannel(options)
try:
convertChannel.receive()
vmWareDir = 'clovr-vmware.%s' % options('general.version')
runSystemEx('mkdir -p ' + vmWareDir, log=True)
runSystemEx('mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join(vmWareDir, 'clovr.9-04.x86-64.%s.vmdk' % options('general.version')))
runSystemEx('mkdir -p %s %s' % (os.path.join(vmWareDir, 'keys'),
os.path.join(vmWareDir, 'user_data')), log=True)
runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir, log=True)
fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w')
clovrConf = config.configFromMap(dict(version=options('general.version')))
for line in open('/usr/local/projects/clovr/start_clovr.vmx'):
fout.write(config.replaceStr(line, clovrConf))
except Exception, err:
errorPrint('Converting image failed. Error message:')
errorPrint(str(err))
def parseInstanceLine(line):
if line.startswith("INSTANCE"):
try:
sline = line.strip().split("\t")
_, instanceId, amiId, pubDns, privDns, state, key, index, _unsure, t, launch, zone, monitor = sline[:13]
return Instance(
instanceId,
amiId,
pubDns,
privDns,
state,
key,
index,
t,
launch,
zone,
monitor,
## Ignoring spot instance stuff for right now
None,
None,
)
except ValueError:
logging.errorPrint("Failed to parse line: " + line)
return None
elif line.startswith("SPOTINSTANCEREQUEST"):
try:
sline = line.split("\t")
_, sir, bid, sType, _arch, _active, _time, _1, _2, _3, _4, instanceId, amiId, iType, key = sline[:15]
return Instance(instanceId, amiId, None, None, None, key, None, iType, None, None, None, sir, bid)
except ValueError:
logging.errorPrint("Failed to parse line: " + line)
return None
else:
return None
def main(options):
conf = configFromStream(open('/tmp/machine.conf'))
runOnElements(conf, '-ss', '-ds')
runOnElements(conf, '-sql', '-dq')
try:
runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf -kej ${MY_IP}', None, None)
except ProgramRunError, err:
errorPrint(str(err))
def runOnElements(conf, query, exc):
try:
outp = []
runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf ' + query, outp.append, None)
hosts = ' '.join(outp)
for h in hosts.split():
runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf %s %s' %(exc, h), None, None)
except ProgramRunError, err:
errorPrint(str(err))
def main(options, _args):
##
# Incredible hack right now
sys.argv = [sys.argv[0]] + ['--help']
try:
pipeline = namedModule('vappio.pipelines.' +
options('general.pipeline'))
runPipeline(None, None, pipeline)
except ImportError:
errorPrint('The requested pipeline could not be found')
def main(options):
conf = configFromStream(open('/tmp/machine.conf'))
runOnElements(conf, '-ss', '-ds')
runOnElements(conf, '-sql', '-dq')
try:
runSingleProgramEx(conf,
'${sge.root}/bin/${sge.arch}/qconf -kej ${MY_IP}',
None, None)
except ProgramRunError, err:
errorPrint(str(err))
def runOnElements(conf, query, exc):
try:
outp = []
runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf ' + query,
outp.append, None)
hosts = ' '.join(outp)
for h in hosts.split():
runSingleProgramEx(
conf, '${sge.root}/bin/${sge.arch}/qconf %s %s' % (exc, h),
None, None)
except ProgramRunError, err:
errorPrint(str(err))
def main(options, _args):
runSystemEx(
'svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"'
% (options('general.version'), options('general.version')),
log=True)
runSystemEx(
'svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"'
% (options('general.version'), options('general.version')),
log=True)
runSystemEx('scp %s:/export/%s .' %
(options('general.remote_name'), options('general.image')),
log=True)
runSystemEx('cp %s /usr/local/projects/clovr/images' %
options('general.image'),
log=True)
runSystemEx('cp %s VMware_conversion/shared/convert_img.img' %
options('general.image'),
log=True)
convertChannel = threads.runThreadWithChannel(
convertImage).channel.sendWithChannel(options)
waitForPasswordChange()
bundleChannel = threads.runThreadWithChannel(
bundleAMI).channel.sendWithChannel(options)
try:
convertChannel.receive()
vmWareDir = 'clovr-vmware.%s' % options('general.version')
runSystemEx('mkdir -p ' + vmWareDir, log=True)
runSystemEx(
'mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join(
vmWareDir,
'clovr.9-04.x86-64.%s.vmdk' % options('general.version')))
runSystemEx('mkdir -p %s %s' % (os.path.join(
vmWareDir, 'keys'), os.path.join(vmWareDir, 'user_data')),
log=True)
runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir,
log=True)
fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w')
clovrConf = config.configFromMap(
dict(version=options('general.version')))
for line in open('/usr/local/projects/clovr/start_clovr.vmx'):
fout.write(config.replaceStr(line, clovrConf))
except Exception, err:
errorPrint('Converting image failed. Error message:')
errorPrint(str(err))
def checkout(self, options, repo, repoPath, outputPath, branch):
fullPath = os.path.join(repo.repoUrl, branch, repoPath)
stderr = []
try:
# Test to see if it's already checked out, if not continue
self._raiseIfCheckout(outputPath)
commands.runSingleProgramEx('svn co %s %s' %
(fullPath, outputPath),
stdoutf=None,
stderrf=stderr.append,
log=logging.DEBUG)
except CheckoutModifiedError:
logging.errorPrint(
'You have uncommited changes to %s, please revert changes or commit them'
% outputPath)
raise
except commands.ProgramRunError:
if 'refers to a file, not a directory' in ''.join(stderr):
try:
tmpPath = os.path.dirname(
os.path.join(options('general.codir'), repoPath))
self._raiseIfCheckout(tmpPath)
commands.runSystem('rm -rf ' + tmpPath, log=logging.DEBUG)
commands.runSystemEx('mkdir -p ' + tmpPath,
log=logging.DEBUG)
commands.runSingleProgramEx(
'svn co %s %s' % (os.path.dirname(fullPath), tmpPath),
stdoutf=None,
stderrf=logging.errorPrintS,
log=logging.DEBUG)
commands.runSystem('rm -rf ' + outputPath,
log=logging.DEBUG)
commands.runSystemEx('ln -s %s %s' % (os.path.join(
options('general.codir'), repoPath), outputPath),
log=logging.DEBUG)
except CheckoutModifiedError:
logging.errorPrint(
'You have uncommited changes to %s, please revert changes or commit them'
% tmpPath)
raise
else:
for l in stderr:
logging.errorPrintS(l)
raise
logExport(options('general.config_dir'), repo, repoPath, outputPath,
branch, CHECKOUT)
def parseInstanceLine(line):
if line.startswith('INSTANCE'):
try:
sline = line.strip().split('\t')
_, instanceId, amiId, pubDns, privDns, state, key, index, _unsure, t, launch, zone, monitor = sline[:13]
return Instance(instanceId,
amiId,
pubDns,
privDns,
state,
key,
index,
t,
launch,
zone,
monitor,
## Ignoring spot instance stuff for right now
None,
None)
except ValueError:
logging.errorPrint('Failed to parse line: ' + line)
return None
elif line.startswith('SPOTINSTANCEREQUEST'):
try:
sline = line.split('\t')
_, sir, bid, sType, _arch, _active, _time, _1, _2, _3, _4, instanceId, amiId, iType, key = sline[:15]
return Instance(instanceId,
amiId,
None,
None,
None,
key,
None,
iType,
None,
None,
None,
sir,
bid)
except ValueError:
logging.errorPrint('Failed to parse line: ' + line)
return None
else:
return None
def uploadTag(srcCluster, dstCluster, tagName, tagData):
"""
srcCluster - Source cluster - currently this needs to be 'local'
dstCluster - Destination cluster
tagName - The tag to be copied, will have the same name on the destination cluster,
must exist on srcCluster
Tags are upload into dstCluster['config']['dirs.upload_dir']
This returns a list of file names that were uploaded
"""
tagBaseDir = tagData('metadata.tag_base_dir', default='')
##
# First step is to create a list of directorys that we should make on
# the destination cluster. We also want to strip off our local dirs.tag_dir
# from the dir names and add teh dstCluster's
dirNames = set([
os.path.join(
dstCluster['config']['dirs.upload_dir'], tagName,
makePathRelative(os.path.dirname(f).replace(tagBaseDir, '')))
for f in tagData('files')
])
##
# Next we want to take the list of local files, remove the dirs.tag_dir and replace it
# with the destination clusters. We maek a list of tuples so we know the local file
# and destinatio file name which we will then loop over and upload
dstFileNames = [(f,
os.path.join(dstCluster['config']['dirs.upload_dir'],
tagName,
makePathRelative(f.replace(tagBaseDir, ''))))
for f in tagData('files')]
##
# Next, lets call 'mkdir -p' on all the dirs we need to make on the destination cluster
try:
makeDirsOnCluster(dstCluster, dirNames)
except TryError, err:
errorPrint('Caught TryError, ignoring for now: %s - %s ' %
(err.msg, str(err.result)))
def downloadUrls(chan):
(options, queue), rchan = chan.receive()
##
# Loop until queue is empty
try:
while True:
url, md5 = queue.get_nowait()
##
# Skip all this if it's already been downloaded
if md5 and validMD5(options, url, md5):
rchan.send((url, True))
continue
if not options('general.continue_download'):
logging.debugPrint(lambda : 'Deleting any files that already exist')
deleteDownloadedFiles(options('general.base_dir'), url)
time.sleep(1)
tries = options('general.tries')
try:
while (not attemptDownload(options, url) or not validMD5(options, url, md5)) and tries > 0:
logging.debugPrint(lambda : 'Download failed, trying again. %d' % tries)
if not options('general.continue_download'):
logging.debugPrint(lambda : 'Deleting downloaded files')
deleteDownloadedFiles(options('general.base_dir'), url)
time.sleep(1)
tries -= 1
if tries <= 0:
rchan.send((url, False))
else:
rchan.send((url, True))
except Exception, err:
logging.errorPrint('Download failed: ' + str(err))
rchan.send((url, False))
except Queue.Empty:
rchan.send(None)
def downloadUrls(chan):
(options, queue), rchan = chan.receive()
##
# Loop until queue is empty
try:
while True:
url, md5 = queue.get_nowait()
##
# Skip all this if it's already been downloaded
if md5 and validMD5(options, url, md5):
rchan.send((url, True))
continue
if not options("general.continue_download"):
logging.debugPrint(lambda: "Deleting any files that already exist")
deleteDownloadedFiles(options("general.base_dir"), url)
time.sleep(1)
tries = options("general.tries")
try:
while (not attemptDownload(options, url) or not validMD5(options, url, md5)) and tries > 0:
logging.debugPrint(lambda: "Download failed, trying again. %d" % tries)
if not options("general.continue_download"):
logging.debugPrint(lambda: "Deleting downloaded files")
deleteDownloadedFiles(options("general.base_dir"), url)
time.sleep(1)
tries -= 1
if tries <= 0:
rchan.send((url, False))
else:
rchan.send((url, True))
except Exception, err:
logging.errorPrint("Download failed: " + str(err))
rchan.send((url, False))
except Queue.Empty:
rchan.send(None)
def executePolicyDirWEx(func, d, prefix=None):
"""
This runs executePolicyDir but catches the TryError and prints out
diagnostic information. and returns safely
"""
try:
executePolicyDir(func, d, prefix)
except TryError, err:
errorPrint('Failed to execute some modules')
for m, e, s in err.result:
errorPrint('Module: %15s Error: %s' % (m, e))
errorPrint('Stacktrace:')
for l in s.splitlines():
errorPrint(l)
def uploadTag(srcCluster, dstCluster, tagName, tagData):
"""
srcCluster - Source cluster - currently this needs to be 'local'
dstCluster - Destination cluster
tagName - The tag to be copied, will have the same name on the destination cluster,
must exist on srcCluster
Tags are upload into dstCluster['config']['dirs.upload_dir']
This returns a list of file names that were uploaded
"""
tagBaseDir = tagData('metadata.tag_base_dir', default='')
##
# First step is to create a list of directorys that we should make on
# the destination cluster. We also want to strip off our local dirs.tag_dir
# from the dir names and add teh dstCluster's
dirNames = set([os.path.join(dstCluster['config']['dirs.upload_dir'], tagName,
makePathRelative(os.path.dirname(f).replace(tagBaseDir, '')))
for f in tagData('files')])
##
# Next we want to take the list of local files, remove the dirs.tag_dir and replace it
# with the destination clusters. We maek a list of tuples so we know the local file
# and destinatio file name which we will then loop over and upload
dstFileNames = [(f, os.path.join(dstCluster['config']['dirs.upload_dir'], tagName,
makePathRelative(f.replace(tagBaseDir, ''))))
for f in tagData('files')]
##
# Next, lets call 'mkdir -p' on all the dirs we need to make on the destination cluster
try:
makeDirsOnCluster(dstCluster, dirNames)
except TryError, err:
errorPrint('Caught TryError, ignoring for now: %s - %s ' % (err.msg, str(err.result)))
def checkout(self, options, repo, repoPath, outputPath, branch):
fullPath = os.path.join(repo.repoUrl, branch, repoPath)
stderr = []
try:
# Test to see if it's already checked out, if not continue
self._raiseIfCheckout(outputPath)
commands.runSingleProgramEx('svn co %s %s' % (fullPath, outputPath),
stdoutf=None,
stderrf=stderr.append,
log=logging.DEBUG)
except CheckoutModifiedError:
logging.errorPrint('You have uncommited changes to %s, please revert changes or commit them' % outputPath)
raise
except commands.ProgramRunError:
if 'refers to a file, not a directory' in ''.join(stderr):
try:
tmpPath = os.path.dirname(os.path.join(options('general.codir'), repoPath))
self._raiseIfCheckout(tmpPath)
commands.runSystem('rm -rf ' + tmpPath, log=logging.DEBUG)
commands.runSystemEx('mkdir -p ' + tmpPath, log=logging.DEBUG)
commands.runSingleProgramEx('svn co %s %s' % (os.path.dirname(fullPath), tmpPath),
stdoutf=None,
stderrf=logging.errorPrintS,
log=logging.DEBUG)
commands.runSystem('rm -rf ' + outputPath, log=logging.DEBUG)
commands.runSystemEx('ln -s %s %s' % (os.path.join(options('general.codir'), repoPath),
outputPath),
log=logging.DEBUG)
except CheckoutModifiedError:
logging.errorPrint('You have uncommited changes to %s, please revert changes or commit them' % tmpPath)
raise
else:
for l in stderr:
logging.errorPrintS(l)
raise
logExport(options('general.config_dir'), repo, repoPath, outputPath, branch, CHECKOUT)
try:
convertChannel.receive()
vmWareDir = 'clovr-vmware.%s' % options('general.version')
runSystemEx('mkdir -p ' + vmWareDir, log=True)
runSystemEx('mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join(vmWareDir, 'clovr.9-04.x86-64.%s.vmdk' % options('general.version')))
runSystemEx('mkdir -p %s %s' % (os.path.join(vmWareDir, 'keys'),
os.path.join(vmWareDir, 'user_data')), log=True)
runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir, log=True)
fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w')
clovrConf = config.configFromMap(dict(version=options('general.version')))
for line in open('/usr/local/projects/clovr/start_clovr.vmx'):
fout.write(config.replaceStr(line, clovrConf))
except Exception, err:
errorPrint('Converting image failed. Error message:')
errorPrint(str(err))
try:
amiId = bundleChannel.receive()
logPrint('AMI: ' + amiId)
except Exception, err:
amiId = None
errorPrint('Bundling AMI failed for some reason. Error message:')
errorPrint(str(err))
if __name__ == '__main__':
main(*buildConfigN(OPTIONS))
def runOnElements(conf, query, exc):
try:
outp = []
runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf ' + query, outp.append, None)
hosts = ' '.join(outp)
for h in hosts.split():
runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf %s %s' %(exc, h), None, None)
except ProgramRunError, err:
errorPrint(str(err))
def main(options):
conf = configFromStream(open('/tmp/machine.conf'))
runOnElements(conf, '-ss', '-ds')
runOnElements(conf, '-sql', '-dq')
try:
runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf -kej ${MY_IP}', None, None)
except ProgramRunError, err:
errorPrint(str(err))
runOnElements(conf, '-sel', '-de')
runOnElements(conf, '-sh', '-dh')
try:
runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf -dprj global', None, None)
except ProgramRunError, err:
errorPrint(str(err))
if __name__ == '__main__':
main(None)
def installAllStow():
for p in [d for d in os.listdir('/usr/local/stow') if d[0] != '.']:
try:
installPkg(p)
except:
logging.errorPrint('Failed to install package: ' + p)
def installAllOptPackages():
for p in [d for d in os.listdir('/opt/opt-packages') if d[0] != '.']:
try:
installOptPkg(p)
except:
logging.errorPrint('Failed to install package: ' + p)
def downloadTag(srcCluster, dstCluster, tagName, dstDir=None, baseDir=None):
"""
srcCluster - Cluster to download the tag from
dstCluster - Cluster to download the tag to (this needs to be 'local' for now)
tagName - The name of the tag to download
dstDir - The destination directory to put downloaded data to. If None, dstDir
is assumed to be dstCluster['config']['dirs.upload_dir']
baseDir - When we download a tag we replicate its directory structure, baseDir allows
us to remove some portion of the prefix dir in downloading. If baseDir is None
then srcCluster['config']['dirs.upload_dir'] is assumed to be the baseDir
Neither dstDir or baseDir should consider the 'tagname' as part of their name.
This may change in the future though if we want to allow downloading to a new tag name
TODO: Consider compressing tags on the remote side before a transfer. It is possible it
should be part of the download process. Alternatively it might make more sense for the compression
to be part of another process.
"""
if dstDir is None:
dstDir = os.path.join(dstCluster['config']['dirs.upload_dir'], tagName)
if baseDir is None:
baseDir = srcCluster['config']['dirs.upload_dir']
#
# Get the list of files
tagData = queryTag('localhost', srcCluster['cluster_name'], [tagName])
#
# Some tags have a tag_base_dir metadata element which shows how much of the path
# to cut off when transfering. However, it is not guaranteed that every file
# in the tag will be in thise base directory, it is meant more as a guide.
# If this metadata element exists we want to group the download into 2 steps.
# 1) Download all files that exist in the tag_base_dir and cut off tag_base_dir
# in the download
# 2) The rest
baseDirFiles, nonBaseDirFiles = partitionFiles(tagData('files'), baseDir)
if baseDirFiles:
#
# Write those files out to a temporary file so we can use it with --files-from in rysnc
tmpFName = os.path.join(dstCluster['config']['general.secure_tmp'],
'rsync-tmp-' + str(time.time()))
fout = open(tmpFName, 'w')
fout.writelines([f + '\n' for f in baseDirFiles])
fout.close()
#
# Do the rsync
try:
makeDirsOnCluster(dstCluster, [dstDir])
except TryError, err:
errorPrint('Caught TryError, ignoring for now: %s - %s ' %
(err.msg, str(err.result)))
rsyncOptions = ' '.join([
dstCluster['config']['rsync.options'], '--files-from=' + tmpFName
])
rsyncFromEx(srcCluster['master']['public_dns'],
baseDir,
dstDir,
rsyncOptions=rsyncOptions,
user=srcCluster['config']['rsync.user'],
log=True)
os.unlink(tmpFName)
def _run(state, batchState):
if 'pipeline_name' in batchState:
pipelines = yield pipelines_client.pipelineList(
'localhost',
'local',
'guest',
batchState['pipeline_name'],
detail=True)
else:
pipelines = []
if not pipelines:
_log(batchState,
'First time running, creating pipeline state information')
batchState['pipeline_config'] = yield _applyActions(
state.innerPipelineConfig(), batchState['actions'])
batchState['pipeline_state'] = STARTCLUSTER_STATE
# We need to create a fake, local, pipeline for metrics to work
batchState['pipeline_name'] = pipeline_misc.checksumInput(
batchState['pipeline_config'])
batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState[
'pipeline_name']
batchState['pipeline_config'][
'pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name']
_log(batchState, 'Pipeline named ' + batchState['pipeline_name'])
pipeline = yield pipelines_client.createPipeline(
host='localhost',
clusterName='local',
userName='******',
pipelineName=batchState['pipeline_name'],
protocol='clovr_wrapper',
queue='pipeline.q',
config=batchState['pipeline_config'],
parentPipeline=state.parentPipeline())
batchState['lgt_wrapper_task_name'] = pipeline['task_name']
_log(
batchState,
'Setting number of tasks to 9 (number in a standard lgt_wrapper)')
yield _updateTask(batchState,
lambda t: t.update(completedTasks=0, numTasks=9))
state.updateBatchState()
batchState['state'] = RUNNING_STATE
_log(batchState,
'Pipeline started in %s state' % batchState['pipeline_state'])
yield _updateTask(batchState,
lambda t: t.setState(tasks.task.TASK_RUNNING))
pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs',
global_state.make_ref() + '.conf')
_log(batchState, 'Creating ergatis configuration')
_writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile)
if batchState['pipeline_state'] == STARTCLUSTER_STATE:
_log(batchState, 'Pipeline is in STARTCLUSTER state')
# First see if the cluster exists but is unresponsive
try:
cluster = yield loadCluster(
'localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest')
if cluster['state'] == 'unresponsive':
_log(batchState, 'Pipeline is unresponsive, terminating')
terminateTask = yield clusters_client.terminateCluster(
'localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
yield _blockOnTask(terminateTask)
except:
pass
batchState['cluster_task'] = yield startCluster(
batchState, 'localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest',
int(batchState['pipeline_config']['cluster.EXEC_NODES']), 0,
batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'], {
'cluster.MASTER_INSTANCE_TYPE':
batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'],
'cluster.MASTER_BID_PRICE':
batchState['pipeline_config']['cluster.MASTER_BID_PRICE'],
'cluster.EXEC_INSTANCE_TYPE':
batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'],
'cluster.EXEC_BID_PRICE':
batchState['pipeline_config']['cluster.EXEC_BID_PRICE']
})
yield _updateTask(batchState,
lambda t: t.update(completedTasks=0, numTasks=9))
yield _updateTask(
batchState, lambda t: t.addMessage(
tasks.task.MSG_SILENT, 'Completed startcluster').progress())
batchState['pipeline_state'] = REMOTE_LOCAL_TRANSFER_STATE
state.updateBatchState()
if batchState['pipeline_state'] == REMOTE_LOCAL_TRANSFER_STATE:
_log(batchState, 'Pipeline is in REMOTE_LOCAL_TRANSFER')
_log(batchState, 'Making sure cluster is exists in some form')
cluster = yield clusters_client.loadCluster(
'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
if cluster['state'] == 'unresponsive':
_log(batchState,
'Pipeline is unresponsive, erroring and restarting')
raise Exception('Cluster is not responsive')
yield state.prerunQueue.addWithDeferred(_remoteLocalTransfer,
batchState)
yield _updateTask(
batchState, lambda t: t.addMessage(
tasks.task.MSG_SILENT, 'Completed remote_local_transfer').
progress())
batchState['pipeline_state'] = DECRYPT_STATE
state.updateBatchState()
if batchState['pipeline_state'] == DECRYPT_STATE:
_log(batchState, 'Pipeline is in DECRYPT')
cluster = yield loadCluster(
'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
tag = yield tags_client.loadTag(
'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest', _decryptTagName(batchState))
conf = config.configFromStream(open('/tmp/machine.conf'))
yield ssh.runProcessSSH(cluster['master']['public_dns'],
'mkdir -p /mnt/lgt_decrypt',
stdoutf=None,
stderrf=None,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
log=True)
yield rsync.rsyncTo(
cluster['master']['public_dns'],
batchState['pipeline_config']['params.DECRYPT_SCRIPT'],
'/mnt/',
options=conf('rsync.options'),
user=conf('rsync.user'),
log=True)
for f in tag['files']:
decryptCmd = ' '.join([
os.path.join(
'/mnt',
os.path.basename(batchState['pipeline_config']
['params.DECRYPT_SCRIPT'])), f,
'-out-dir', '/mnt/lgt_decrypt', '-remove-encrypted',
'-password',
batchState['pipeline_config']['params.DECRYPT_PASSWORD']
])
yield ssh.getOutput(cluster['master']['public_dns'],
decryptCmd,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
expected=[0, 253],
log=True)
tag = yield tags_client.tagData(
host='localhost',
clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'],
userName='******',
action='overwrite',
tagName=_decryptTagName(batchState),
files=['/mnt/lgt_decrypt'],
metadata={},
recursive=True,
expand=False,
compressDir=None)
_log(
batchState, 'Waiting for tagging of %s to complete - %s' %
(_decryptTagName(batchState), tag['task_name']))
yield _blockOnTask(
tag['task_name'],
cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME'])
yield _updateTask(
batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT,
'Completed decrypt').progress())
batchState['pipeline_state'] = REFERENCE_TRANSFER_STATE
state.updateBatchState()
if batchState['pipeline_state'] == REFERENCE_TRANSFER_STATE:
_log(batchState, 'Pipeline is in REFERENCE_TRANSFER state')
transfers = []
tags = (batchState['pipeline_config']['input.REF_TAG1'].split(',') +
batchState['pipeline_config']['input.REF_TAG2'].split(','))
for tag in tags:
tag = tag.strip()
output = yield _getOutput(batchState, [
'vp-transfer-dataset', '-t', '--tag-name=' + tag,
'--dst-cluster=' +
batchState['pipeline_config']['cluster.CLUSTER_NAME']
],
log=True)
transfers.append(output['stdout'].strip())
for task in transfers:
yield _blockOnTask(task)
yield _updateTask(
batchState,
lambda t: t.addMessage(tasks.task.MSG_SILENT,
'Completed reference_transfer').progress())
batchState['pipeline_state'] = RUN_PIPELINE_STATE
state.updateBatchState()
if batchState['pipeline_state'] == RUN_PIPELINE_STATE:
_log(batchState, 'Pipeline is in RUN_PIPELINE state')
batchState['pipeline_config']['input.INPUT_TAG'] = _decryptTagName(
batchState)
pipeline = yield pipelines_client.runPipeline(
host='localhost',
clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'],
userName='******',
parentPipeline=batchState['pipeline_name'],
bareRun=True,
queue=state.innerPipelineQueue(),
config=batchState['pipeline_config'],
overwrite=True)
batchState['pipeline_task'] = pipeline['task_name']
yield _updateTask(
batchState, lambda t: t.addMessage(
tasks.task.MSG_SILENT, 'Completed run pipeline').progress())
batchState['pipeline_state'] = RUNNING_PIPELINE_STATE
state.updateBatchState()
if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE:
_log(batchState, 'Pipeline is in RUNNING_PIPELINE state')
_monitorPipeline(batchState)
yield _waitForPipeline(batchState)
yield _updateTask(
batchState,
lambda t: t.addMessage(tasks.task.MSG_SILENT,
'Completed running pipeline').progress())
batchState['pipeline_state'] = HARVEST_STATE
state.updateBatchState()
if batchState['pipeline_state'] == HARVEST_STATE:
_log(batchState, 'Pipeline is in HARVEST state')
# Using prerunqueue because we want everything here serialized
yield state.prerunQueue.addWithDeferred(_harvestTransfer, batchState)
yield _updateTask(
batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT,
'Completed harvest').progress())
batchState['pipeline_state'] = SHUTDOWN_STATE
state.updateBatchState()
if batchState['pipeline_state'] == SHUTDOWN_STATE:
_log(batchState, 'Pipeline is in SHUTDOWN state')
if 'add_instances_task' in batchState:
try:
yield _blockOnTask(batchState['add_instances_task'],
cluster=batchState['pipeline_config']
['cluster.CLUSTER_NAME'])
except Exception, err:
logging.errorPrint(str(err))
log.err(err)
yield clusters_client.terminateCluster(
'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
yield _updateTask(
batchState, lambda t: t.addMessage(
tasks.task.MSG_SILENT, 'Completed shutdown').progress())
batchState['pipeline_state'] = COMPLETED_STATE
batchState['state'] = COMPLETED_STATE
state.updateBatchState()
def main(options, args):
logging.DEBUG = options('general.debug')
queue = Queue.Queue()
##
# Track the downloaded URL names for joining later if specified
urls = []
if not args:
for line in [l for l in sys.stdin if l.strip()]:
md5, url = line.split(' ', 1)
url = url.strip()
urls.append(url)
queue.put((url, md5))
else:
for url in args:
urls.append(url)
queue.put((url, None))
if options('general.join_name') and options('general.join_md5'):
md5 = calculateMD5([os.path.join(options('general.base_dir'), options('general.join_name'))])
##
# If they match, then no need to download and exit cleanly
if md5 == options('general.join_md5'):
return
retChans = [threads.runThreadWithChannel(downloadUrls).channel.sendWithChannel((options, queue)) for _ in range(options('general.max_threads'))]
successUrls = []
failedUrls = []
for c in retChans:
ret = c.receive()
while ret is not None:
url, succ = ret
if succ:
successUrls.append(url)
else:
failedUrls.append(url)
ret = c.receive()
if failedUrls:
for url in failedUrls:
logging.errorPrint(url)
##
# If any URLs failed, exit with fail
sys.exit(1)
else:
if options('general.join_name'):
logging.debugPrint(lambda : 'Joining files into: ' + options('general.join_name'))
files = []
for url in urls:
files.extend(sorted(getDownloadFilenames(options('general.base_dir'), url)))
fout = open(os.path.join(options('general.base_dir'), options('general.join_name')), 'wb')
for f in files:
logging.debugPrint(lambda : 'Reading: ' + f)
fin = open(f, 'rb')
d = fin.read(1000000)
while d:
fout.write(d)
d = fin.read(1000000)
fin.close()
fout.close()
logging.debugPrint(lambda : 'Deleting downloaded files after join')
for f in files:
logging.debugPrint(lambda : 'Deleting: ' + f)
deleteFile(f)
def main(options, _args):
updateAll = False
for o in OPTIONS:
if o[0] not in ['vappio_branch', 'clovr_branch'
] and options('general.' + o[0]):
break
else:
updateAll = True
clovrBranch = options('general.clovr_branch')
vappioBranch = options('general.vappio_branch')
try:
if options('general.stow') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code',
clovrBranch, 'stow', '/usr/local/stow')
if options('general.opt_packages') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code',
clovrBranch, 'opt-packages', '/opt/opt-packages')
if options('general.config_policies') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code',
clovrBranch, 'config_policies', '/opt/config_policies')
if options('general.vappio_py') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code',
vappioBranch, 'vappio-py', '/opt/vappio-py')
runSystemEx("""chmod +x /opt/vappio-py/vappio/cli/*.py""")
if options('general.vappio_apps') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code',
vappioBranch, 'vappio-apps', '/opt/vappio-apps')
if options('general.vappio_twisted') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code',
vappioBranch, 'vappio-twisted', '/opt/vappio-twisted')
if options('general.vappio_scripts') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code',
vappioBranch, 'vappio-scripts', '/opt/vappio-scripts')
runSystemEx("""chmod -R +x /opt/vappio-scripts""", log=True)
runSystemEx("""cp -f /opt/vappio-scripts/clovrEnv.sh /root""",
log=True)
runSystemEx(
"""cp -f /opt/vappio-scripts/local /etc/init.d/local""",
log=True)
runSystemEx(
"""cp -f /opt/vappio-scripts/rc.local /etc/init.d/rc.local""",
log=True)
runSystemEx(
"""cp -f /opt/vappio-scripts/screenrc /root/.screenrc""",
log=True)
if options('general.clovr_pipelines') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code',
clovrBranch, 'clovr_pipelines', '/opt/clovr_pipelines')
if options('general.vappio_py_www') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code',
vappioBranch, 'vappio-www/py-www', '/var/www/vappio')
##
# Only want to do this one when specified
# if options('general.vappio_conf'):
# grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code', vappioBranch, 'vappio-conf', '/mnt/vappio-conf')
if options('general.hudson') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code',
clovrBranch, 'hudson/hudson-config/jobs',
'/var/lib/hudson/jobs')
grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code',
clovrBranch, 'hudson/hudson-scripts', '/opt/hudson')
runSystemEx("""chown -R hudson.nogroup /var/lib/hudson/jobs""",
log=True)
if options('general.clovr_www') or updateAll:
grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code',
clovrBranch, 'clovr-www', '/var/www/clovr')
except CheckoutModifiedError, err:
errorPrint(str(err))
for h in hosts.split():
runSingleProgramEx(
conf, '${sge.root}/bin/${sge.arch}/qconf %s %s' % (exc, h),
None, None)
except ProgramRunError, err:
errorPrint(str(err))
def main(options):
conf = configFromStream(open('/tmp/machine.conf'))
runOnElements(conf, '-ss', '-ds')
runOnElements(conf, '-sql', '-dq')
try:
runSingleProgramEx(conf,
'${sge.root}/bin/${sge.arch}/qconf -kej ${MY_IP}',
None, None)
except ProgramRunError, err:
errorPrint(str(err))
runOnElements(conf, '-sel', '-de')
runOnElements(conf, '-sh', '-dh')
try:
runSingleProgramEx(conf,
'${sge.root}/bin/${sge.arch}/qconf -dprj global',
None, None)
except ProgramRunError, err:
errorPrint(str(err))
if __name__ == '__main__':
main(None)
def installAllStow():
for p in [d for d in os.listdir('/usr/local/stow') if d[0] != '.']:
try:
installPkg(p)
except:
logging.errorPrint('Failed to install package: ' + p)
def installAllOptPackages():
for p in [d for d in os.listdir('/opt/opt-packages') if d[0] != '.']:
try:
installOptPkg(p)
except:
logging.errorPrint('Failed to install package: ' + p)
def main(options, args):
logging.DEBUG = options("general.debug")
queue = Queue.Queue()
##
# Track the downloaded URL names for joining later if specified
urls = []
if not args:
for line in [l for l in sys.stdin if l.strip()]:
md5, url = line.split(" ", 1)
url = url.strip()
urls.append(url)
queue.put((url, md5))
else:
for url in args:
urls.append(url)
queue.put((url, None))
if options("general.join_name") and options("general.join_md5"):
md5 = calculateMD5([os.path.join(options("general.base_dir"), options("general.join_name"))])
##
# If they match, then no need to download and exit cleanly
if md5 == options("general.join_md5"):
return
retChans = [
threads.runThreadWithChannel(downloadUrls).channel.sendWithChannel((options, queue))
for _ in range(options("general.max_threads"))
]
successUrls = []
failedUrls = []
for c in retChans:
ret = c.receive()
while ret is not None:
url, succ = ret
if succ:
successUrls.append(url)
else:
failedUrls.append(url)
ret = c.receive()
if failedUrls:
for url in failedUrls:
logging.errorPrint(url)
##
# If any URLs failed, exit with fail
sys.exit(1)
else:
if options("general.join_name"):
logging.debugPrint(lambda: "Joining files into: " + options("general.join_name"))
files = []
for url in urls:
files.extend(sorted(getDownloadFilenames(options("general.base_dir"), url)))
fout = open(os.path.join(options("general.base_dir"), options("general.join_name")), "wb")
for f in files:
logging.debugPrint(lambda: "Reading: " + f)
fin = open(f, "rb")
d = fin.read(1000000)
while d:
fout.write(d)
d = fin.read(1000000)
fin.close()
fout.close()
logging.debugPrint(lambda: "Deleting downloaded files after join")
for f in files:
logging.debugPrint(lambda: "Deleting: " + f)
deleteFile(f)
runSystemEx('mkdir -p ' + vmWareDir, log=True)
runSystemEx(
'mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join(
vmWareDir,
'clovr.9-04.x86-64.%s.vmdk' % options('general.version')))
runSystemEx('mkdir -p %s %s' % (os.path.join(
vmWareDir, 'keys'), os.path.join(vmWareDir, 'user_data')),
log=True)
runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir,
log=True)
fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w')
clovrConf = config.configFromMap(
dict(version=options('general.version')))
for line in open('/usr/local/projects/clovr/start_clovr.vmx'):
fout.write(config.replaceStr(line, clovrConf))
except Exception, err:
errorPrint('Converting image failed. Error message:')
errorPrint(str(err))
try:
amiId = bundleChannel.receive()
logPrint('AMI: ' + amiId)
except Exception, err:
amiId = None
errorPrint('Bundling AMI failed for some reason. Error message:')
errorPrint(str(err))
if __name__ == '__main__':
main(*buildConfigN(OPTIONS))
def _run(state, batchState):
if 'pipeline_name' in batchState:
pipelines = yield pipelines_client.pipelineList('localhost',
'local',
'guest',
batchState['pipeline_name'],
detail=True)
else:
pipelines = []
if not pipelines:
_log(batchState, 'First time running, creating pipeline state information')
batchState['pipeline_config'] = yield _applyActions(state.innerPipelineConfig(),
batchState['actions'])
batchState['pipeline_state'] = STARTCLUSTER_STATE
# We need to create a fake, local, pipeline for metrics to work
batchState['pipeline_name'] = pipeline_misc.checksumInput(batchState['pipeline_config'])
batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState['pipeline_name']
batchState['pipeline_config']['pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name']
_log(batchState, 'Pipeline named ' + batchState['pipeline_name'])
pipeline = yield pipelines_client.createPipeline(host='localhost',
clusterName='local',
userName='******',
pipelineName=batchState['pipeline_name'],
protocol='clovr_wrapper',
queue='pipeline.q',
config=batchState['pipeline_config'],
parentPipeline=state.parentPipeline())
batchState['lgt_wrapper_task_name'] = pipeline['task_name']
_log(batchState, 'Setting number of tasks to 9 (number in a standard lgt_wrapper)')
yield _updateTask(batchState,
lambda t : t.update(completedTasks=0,
numTasks=9))
state.updateBatchState()
batchState['state'] = RUNNING_STATE
_log(batchState, 'Pipeline started in %s state' % batchState['pipeline_state'])
yield _updateTask(batchState,
lambda t : t.setState(tasks.task.TASK_RUNNING))
pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs', global_state.make_ref() + '.conf')
_log(batchState, 'Creating ergatis configuration')
_writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile)
if batchState['pipeline_state'] == STARTCLUSTER_STATE:
_log(batchState, 'Pipeline is in STARTCLUSTER state')
# First see if the cluster exists but is unresponsive
try:
cluster = yield loadCluster('localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
if cluster['state'] == 'unresponsive':
_log(batchState, 'Pipeline is unresponsive, terminating')
terminateTask = yield clusters_client.terminateCluster('localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
yield _blockOnTask(terminateTask)
except:
pass
batchState['cluster_task'] = yield startCluster(
batchState,
'localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest',
int(batchState['pipeline_config']['cluster.EXEC_NODES']),
0,
batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'],
{'cluster.MASTER_INSTANCE_TYPE':
batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'],
'cluster.MASTER_BID_PRICE':
batchState['pipeline_config']['cluster.MASTER_BID_PRICE'],
'cluster.EXEC_INSTANCE_TYPE':
batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'],
'cluster.EXEC_BID_PRICE':
batchState['pipeline_config']['cluster.EXEC_BID_PRICE']})
yield _updateTask(batchState,
lambda t : t.update(completedTasks=0,
numTasks=9))
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed startcluster'
).progress())
batchState['pipeline_state'] = REMOTE_LOCAL_TRANSFER_STATE
state.updateBatchState()
if batchState['pipeline_state'] == REMOTE_LOCAL_TRANSFER_STATE:
_log(batchState, 'Pipeline is in REMOTE_LOCAL_TRANSFER')
_log(batchState, 'Making sure cluster is exists in some form')
cluster = yield clusters_client.loadCluster('localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
if cluster['state'] == 'unresponsive':
_log(batchState, 'Pipeline is unresponsive, erroring and restarting')
raise Exception('Cluster is not responsive')
yield state.prerunQueue.addWithDeferred(_remoteLocalTransfer,
batchState)
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed remote_local_transfer'
).progress())
batchState['pipeline_state'] = DECRYPT_STATE
state.updateBatchState()
if batchState['pipeline_state'] == DECRYPT_STATE:
_log(batchState, 'Pipeline is in DECRYPT')
cluster = yield loadCluster('localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
tag = yield tags_client.loadTag('localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest',
_decryptTagName(batchState))
conf = config.configFromStream(open('/tmp/machine.conf'))
yield ssh.runProcessSSH(cluster['master']['public_dns'],
'mkdir -p /mnt/lgt_decrypt',
stdoutf=None,
stderrf=None,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
log=True)
yield rsync.rsyncTo(cluster['master']['public_dns'],
batchState['pipeline_config']['params.DECRYPT_SCRIPT'],
'/mnt/',
options=conf('rsync.options'),
user=conf('rsync.user'),
log=True)
for f in tag['files']:
decryptCmd = ' '.join([os.path.join('/mnt', os.path.basename(batchState['pipeline_config']['params.DECRYPT_SCRIPT'])),
f,
'-out-dir', '/mnt/lgt_decrypt',
'-remove-encrypted',
'-password', batchState['pipeline_config']['params.DECRYPT_PASSWORD']])
yield ssh.getOutput(cluster['master']['public_dns'],
decryptCmd,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
expected=[0, 253],
log=True)
tag = yield tags_client.tagData(host='localhost',
clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'],
userName='******',
action='overwrite',
tagName=_decryptTagName(batchState),
files=['/mnt/lgt_decrypt'],
metadata={},
recursive=True,
expand=False,
compressDir=None)
_log(batchState, 'Waiting for tagging of %s to complete - %s' % (_decryptTagName(batchState),
tag['task_name']))
yield _blockOnTask(tag['task_name'],
cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME'])
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed decrypt'
).progress())
batchState['pipeline_state'] = REFERENCE_TRANSFER_STATE
state.updateBatchState()
if batchState['pipeline_state'] == REFERENCE_TRANSFER_STATE:
_log(batchState, 'Pipeline is in REFERENCE_TRANSFER state')
transfers = []
tags = (batchState['pipeline_config']['input.REF_TAG1'].split(',') +
batchState['pipeline_config']['input.REF_TAG2'].split(','))
for tag in tags:
tag = tag.strip()
output = yield _getOutput(batchState,
['vp-transfer-dataset',
'-t',
'--tag-name=' + tag,
'--dst-cluster=' + batchState['pipeline_config']['cluster.CLUSTER_NAME']],
log=True)
transfers.append(output['stdout'].strip())
for task in transfers:
yield _blockOnTask(task)
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed reference_transfer'
).progress())
batchState['pipeline_state'] = RUN_PIPELINE_STATE
state.updateBatchState()
if batchState['pipeline_state'] == RUN_PIPELINE_STATE:
_log(batchState, 'Pipeline is in RUN_PIPELINE state')
batchState['pipeline_config']['input.INPUT_TAG'] = _decryptTagName(batchState)
pipeline = yield pipelines_client.runPipeline(host='localhost',
clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'],
userName='******',
parentPipeline=batchState['pipeline_name'],
bareRun=True,
queue=state.innerPipelineQueue(),
config=batchState['pipeline_config'],
overwrite=True)
batchState['pipeline_task'] = pipeline['task_name']
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed run pipeline'
).progress())
batchState['pipeline_state'] = RUNNING_PIPELINE_STATE
state.updateBatchState()
if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE:
_log(batchState, 'Pipeline is in RUNNING_PIPELINE state')
_monitorPipeline(batchState)
yield _waitForPipeline(batchState)
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed running pipeline'
).progress())
batchState['pipeline_state'] = HARVEST_STATE
state.updateBatchState()
if batchState['pipeline_state'] == HARVEST_STATE:
_log(batchState, 'Pipeline is in HARVEST state')
# Using prerunqueue because we want everything here serialized
yield state.prerunQueue.addWithDeferred(_harvestTransfer,
batchState)
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed harvest'
).progress())
batchState['pipeline_state'] = SHUTDOWN_STATE
state.updateBatchState()
if batchState['pipeline_state'] == SHUTDOWN_STATE:
_log(batchState, 'Pipeline is in SHUTDOWN state')
if 'add_instances_task' in batchState:
try:
yield _blockOnTask(batchState['add_instances_task'],
cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME'])
except Exception, err:
logging.errorPrint(str(err))
log.err(err)
yield clusters_client.terminateCluster('localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed shutdown'
).progress())
batchState['pipeline_state'] = COMPLETED_STATE
batchState['state'] = COMPLETED_STATE
state.updateBatchState()
##
# Now, copy up all of the files
for l, d in dstFileNames:
scpToEx(dstCluster['master']['public_dns'], l, d, user=srcCluster['config']['ssh.user'], options=srcCluster['config']['ssh.options'], log=True)
##
# We are uploading as root, so chown everything to the user that everything in vappio will be done under
try:
runSystemInstanceEx(dstCluster['master'],
'chown %s %s' % (dstCluster['config']['vappio.user'], d),
None,
errorPrintS,
user=dstCluster['config']['ssh.user'],
options=dstCluster['config']['ssh.options'],
log=True)
except ProgramRunError:
errorPrint('Chown failed on ' + d)
##
# return the list of uploaded filenames
return [d for l, d in dstFileNames]
def partitionFiles(files, baseDir):
if baseDir:
baseDirFiles = [f.replace(baseDir, '')
for f in files
if f.startswith(baseDir)]
downloadFiles = [f
for f in files
if not f.startswith(baseDir)]
return (baseDirFiles, downloadFiles)
def downloadTag(srcCluster, dstCluster, tagName, dstDir=None, baseDir=None):
"""
srcCluster - Cluster to download the tag from
dstCluster - Cluster to download the tag to (this needs to be 'local' for now)
tagName - The name of the tag to download
dstDir - The destination directory to put downloaded data to. If None, dstDir
is assumed to be dstCluster['config']['dirs.upload_dir']
baseDir - When we download a tag we replicate its directory structure, baseDir allows
us to remove some portion of the prefix dir in downloading. If baseDir is None
then srcCluster['config']['dirs.upload_dir'] is assumed to be the baseDir
Neither dstDir or baseDir should consider the 'tagname' as part of their name.
This may change in the future though if we want to allow downloading to a new tag name
TODO: Consider compressing tags on the remote side before a transfer. It is possible it
should be part of the download process. Alternatively it might make more sense for the compression
to be part of another process.
"""
if dstDir is None:
dstDir = os.path.join(dstCluster['config']['dirs.upload_dir'], tagName)
if baseDir is None:
baseDir = srcCluster['config']['dirs.upload_dir']
#
# Get the list of files
tagData = queryTag('localhost', srcCluster['cluster_name'], [tagName])
#
# Some tags have a tag_base_dir metadata element which shows how much of the path
# to cut off when transfering. However, it is not guaranteed that every file
# in the tag will be in thise base directory, it is meant more as a guide.
# If this metadata element exists we want to group the download into 2 steps.
# 1) Download all files that exist in the tag_base_dir and cut off tag_base_dir
# in the download
# 2) The rest
baseDirFiles, nonBaseDirFiles = partitionFiles(tagData('files'), baseDir)
if baseDirFiles:
#
# Write those files out to a temporary file so we can use it with --files-from in rysnc
tmpFName = os.path.join(dstCluster['config']['general.secure_tmp'], 'rsync-tmp-' + str(time.time()))
fout = open(tmpFName, 'w')
fout.writelines([f + '\n' for f in baseDirFiles])
fout.close()
#
# Do the rsync
try:
makeDirsOnCluster(dstCluster, [dstDir])
except TryError, err:
errorPrint('Caught TryError, ignoring for now: %s - %s ' % (err.msg, str(err.result)))
rsyncOptions = ' '.join([dstCluster['config']['rsync.options'], '--files-from=' + tmpFName])
rsyncFromEx(srcCluster['master']['public_dns'],
baseDir,
dstDir,
rsyncOptions=rsyncOptions,
user=srcCluster['config']['rsync.user'],
log=True)
os.unlink(tmpFName)
user=srcCluster['config']['ssh.user'],
options=srcCluster['config']['ssh.options'],
log=True)
##
# We are uploading as root, so chown everything to the user that everything in vappio will be done under
try:
runSystemInstanceEx(dstCluster['master'],
'chown %s %s' %
(dstCluster['config']['vappio.user'], d),
None,
errorPrintS,
user=dstCluster['config']['ssh.user'],
options=dstCluster['config']['ssh.options'],
log=True)
except ProgramRunError:
errorPrint('Chown failed on ' + d)
##
# return the list of uploaded filenames
return [d for l, d in dstFileNames]
def partitionFiles(files, baseDir):
if baseDir:
baseDirFiles = [
f.replace(baseDir, '') for f in files if f.startswith(baseDir)
]
downloadFiles = [f for f in files if not f.startswith(baseDir)]
return (baseDirFiles, downloadFiles)
else:
return ([], files)
