def main(options, _args): runSystemEx('svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx('svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx('scp %s:/export/%s .' % (options('general.remote_name'), options('general.image')), log=True) runSystemEx('cp %s /usr/local/projects/clovr/images' % options('general.image'), log=True) runSystemEx('cp %s VMware_conversion/shared/convert_img.img' % options('general.image'), log=True) convertChannel = threads.runThreadWithChannel(convertImage).channel.sendWithChannel(options) waitForPasswordChange() bundleChannel = threads.runThreadWithChannel(bundleAMI).channel.sendWithChannel(options) try: convertChannel.receive() vmWareDir = 'clovr-vmware.%s' % options('general.version') runSystemEx('mkdir -p ' + vmWareDir, log=True) runSystemEx('mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join(vmWareDir, 'clovr.9-04.x86-64.%s.vmdk' % options('general.version'))) runSystemEx('mkdir -p %s %s' % (os.path.join(vmWareDir, 'keys'), os.path.join(vmWareDir, 'user_data')), log=True) runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir, log=True) fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w') clovrConf = config.configFromMap(dict(version=options('general.version'))) for line in open('/usr/local/projects/clovr/start_clovr.vmx'): fout.write(config.replaceStr(line, clovrConf)) except Exception, err: errorPrint('Converting image failed. Error message:') errorPrint(str(err))
def parseInstanceLine(line): if line.startswith("INSTANCE"): try: sline = line.strip().split("\t") _, instanceId, amiId, pubDns, privDns, state, key, index, _unsure, t, launch, zone, monitor = sline[:13] return Instance( instanceId, amiId, pubDns, privDns, state, key, index, t, launch, zone, monitor, ## Ignoring spot instance stuff for right now None, None, ) except ValueError: logging.errorPrint("Failed to parse line: " + line) return None elif line.startswith("SPOTINSTANCEREQUEST"): try: sline = line.split("\t") _, sir, bid, sType, _arch, _active, _time, _1, _2, _3, _4, instanceId, amiId, iType, key = sline[:15] return Instance(instanceId, amiId, None, None, None, key, None, iType, None, None, None, sir, bid) except ValueError: logging.errorPrint("Failed to parse line: " + line) return None else: return None
def main(options): conf = configFromStream(open('/tmp/machine.conf')) runOnElements(conf, '-ss', '-ds') runOnElements(conf, '-sql', '-dq') try: runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf -kej ${MY_IP}', None, None) except ProgramRunError, err: errorPrint(str(err))
def runOnElements(conf, query, exc): try: outp = [] runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf ' + query, outp.append, None) hosts = ' '.join(outp) for h in hosts.split(): runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf %s %s' %(exc, h), None, None) except ProgramRunError, err: errorPrint(str(err))
def main(options, _args): ## # Incredible hack right now sys.argv = [sys.argv[0]] + ['--help'] try: pipeline = namedModule('vappio.pipelines.' + options('general.pipeline')) runPipeline(None, None, pipeline) except ImportError: errorPrint('The requested pipeline could not be found')
def runOnElements(conf, query, exc): try: outp = [] runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf ' + query, outp.append, None) hosts = ' '.join(outp) for h in hosts.split(): runSingleProgramEx( conf, '${sge.root}/bin/${sge.arch}/qconf %s %s' % (exc, h), None, None) except ProgramRunError, err: errorPrint(str(err))
def main(options, _args): runSystemEx( 'svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx( 'svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx('scp %s:/export/%s .' % (options('general.remote_name'), options('general.image')), log=True) runSystemEx('cp %s /usr/local/projects/clovr/images' % options('general.image'), log=True) runSystemEx('cp %s VMware_conversion/shared/convert_img.img' % options('general.image'), log=True) convertChannel = threads.runThreadWithChannel( convertImage).channel.sendWithChannel(options) waitForPasswordChange() bundleChannel = threads.runThreadWithChannel( bundleAMI).channel.sendWithChannel(options) try: convertChannel.receive() vmWareDir = 'clovr-vmware.%s' % options('general.version') runSystemEx('mkdir -p ' + vmWareDir, log=True) runSystemEx( 'mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join( vmWareDir, 'clovr.9-04.x86-64.%s.vmdk' % options('general.version'))) runSystemEx('mkdir -p %s %s' % (os.path.join( vmWareDir, 'keys'), os.path.join(vmWareDir, 'user_data')), log=True) runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir, log=True) fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w') clovrConf = config.configFromMap( dict(version=options('general.version'))) for line in open('/usr/local/projects/clovr/start_clovr.vmx'): fout.write(config.replaceStr(line, clovrConf)) except Exception, err: errorPrint('Converting image failed. Error message:') errorPrint(str(err))
def checkout(self, options, repo, repoPath, outputPath, branch): fullPath = os.path.join(repo.repoUrl, branch, repoPath) stderr = [] try: # Test to see if it's already checked out, if not continue self._raiseIfCheckout(outputPath) commands.runSingleProgramEx('svn co %s %s' % (fullPath, outputPath), stdoutf=None, stderrf=stderr.append, log=logging.DEBUG) except CheckoutModifiedError: logging.errorPrint( 'You have uncommited changes to %s, please revert changes or commit them' % outputPath) raise except commands.ProgramRunError: if 'refers to a file, not a directory' in ''.join(stderr): try: tmpPath = os.path.dirname( os.path.join(options('general.codir'), repoPath)) self._raiseIfCheckout(tmpPath) commands.runSystem('rm -rf ' + tmpPath, log=logging.DEBUG) commands.runSystemEx('mkdir -p ' + tmpPath, log=logging.DEBUG) commands.runSingleProgramEx( 'svn co %s %s' % (os.path.dirname(fullPath), tmpPath), stdoutf=None, stderrf=logging.errorPrintS, log=logging.DEBUG) commands.runSystem('rm -rf ' + outputPath, log=logging.DEBUG) commands.runSystemEx('ln -s %s %s' % (os.path.join( options('general.codir'), repoPath), outputPath), log=logging.DEBUG) except CheckoutModifiedError: logging.errorPrint( 'You have uncommited changes to %s, please revert changes or commit them' % tmpPath) raise else: for l in stderr: logging.errorPrintS(l) raise logExport(options('general.config_dir'), repo, repoPath, outputPath, branch, CHECKOUT)
def parseInstanceLine(line): if line.startswith('INSTANCE'): try: sline = line.strip().split('\t') _, instanceId, amiId, pubDns, privDns, state, key, index, _unsure, t, launch, zone, monitor = sline[:13] return Instance(instanceId, amiId, pubDns, privDns, state, key, index, t, launch, zone, monitor, ## Ignoring spot instance stuff for right now None, None) except ValueError: logging.errorPrint('Failed to parse line: ' + line) return None elif line.startswith('SPOTINSTANCEREQUEST'): try: sline = line.split('\t') _, sir, bid, sType, _arch, _active, _time, _1, _2, _3, _4, instanceId, amiId, iType, key = sline[:15] return Instance(instanceId, amiId, None, None, None, key, None, iType, None, None, None, sir, bid) except ValueError: logging.errorPrint('Failed to parse line: ' + line) return None else: return None
def uploadTag(srcCluster, dstCluster, tagName, tagData): """ srcCluster - Source cluster - currently this needs to be 'local' dstCluster - Destination cluster tagName - The tag to be copied, will have the same name on the destination cluster, must exist on srcCluster Tags are upload into dstCluster['config']['dirs.upload_dir'] This returns a list of file names that were uploaded """ tagBaseDir = tagData('metadata.tag_base_dir', default='') ## # First step is to create a list of directorys that we should make on # the destination cluster. We also want to strip off our local dirs.tag_dir # from the dir names and add teh dstCluster's dirNames = set([ os.path.join( dstCluster['config']['dirs.upload_dir'], tagName, makePathRelative(os.path.dirname(f).replace(tagBaseDir, ''))) for f in tagData('files') ]) ## # Next we want to take the list of local files, remove the dirs.tag_dir and replace it # with the destination clusters. We maek a list of tuples so we know the local file # and destinatio file name which we will then loop over and upload dstFileNames = [(f, os.path.join(dstCluster['config']['dirs.upload_dir'], tagName, makePathRelative(f.replace(tagBaseDir, '')))) for f in tagData('files')] ## # Next, lets call 'mkdir -p' on all the dirs we need to make on the destination cluster try: makeDirsOnCluster(dstCluster, dirNames) except TryError, err: errorPrint('Caught TryError, ignoring for now: %s - %s ' % (err.msg, str(err.result)))
def downloadUrls(chan): (options, queue), rchan = chan.receive() ## # Loop until queue is empty try: while True: url, md5 = queue.get_nowait() ## # Skip all this if it's already been downloaded if md5 and validMD5(options, url, md5): rchan.send((url, True)) continue if not options('general.continue_download'): logging.debugPrint(lambda : 'Deleting any files that already exist') deleteDownloadedFiles(options('general.base_dir'), url) time.sleep(1) tries = options('general.tries') try: while (not attemptDownload(options, url) or not validMD5(options, url, md5)) and tries > 0: logging.debugPrint(lambda : 'Download failed, trying again. %d' % tries) if not options('general.continue_download'): logging.debugPrint(lambda : 'Deleting downloaded files') deleteDownloadedFiles(options('general.base_dir'), url) time.sleep(1) tries -= 1 if tries <= 0: rchan.send((url, False)) else: rchan.send((url, True)) except Exception, err: logging.errorPrint('Download failed: ' + str(err)) rchan.send((url, False)) except Queue.Empty: rchan.send(None)
def downloadUrls(chan): (options, queue), rchan = chan.receive() ## # Loop until queue is empty try: while True: url, md5 = queue.get_nowait() ## # Skip all this if it's already been downloaded if md5 and validMD5(options, url, md5): rchan.send((url, True)) continue if not options("general.continue_download"): logging.debugPrint(lambda: "Deleting any files that already exist") deleteDownloadedFiles(options("general.base_dir"), url) time.sleep(1) tries = options("general.tries") try: while (not attemptDownload(options, url) or not validMD5(options, url, md5)) and tries > 0: logging.debugPrint(lambda: "Download failed, trying again. %d" % tries) if not options("general.continue_download"): logging.debugPrint(lambda: "Deleting downloaded files") deleteDownloadedFiles(options("general.base_dir"), url) time.sleep(1) tries -= 1 if tries <= 0: rchan.send((url, False)) else: rchan.send((url, True)) except Exception, err: logging.errorPrint("Download failed: " + str(err)) rchan.send((url, False)) except Queue.Empty: rchan.send(None)
def executePolicyDirWEx(func, d, prefix=None): """ This runs executePolicyDir but catches the TryError and prints out diagnostic information. and returns safely """ try: executePolicyDir(func, d, prefix) except TryError, err: errorPrint('Failed to execute some modules') for m, e, s in err.result: errorPrint('Module: %15s Error: %s' % (m, e)) errorPrint('Stacktrace:') for l in s.splitlines(): errorPrint(l)
def uploadTag(srcCluster, dstCluster, tagName, tagData): """ srcCluster - Source cluster - currently this needs to be 'local' dstCluster - Destination cluster tagName - The tag to be copied, will have the same name on the destination cluster, must exist on srcCluster Tags are upload into dstCluster['config']['dirs.upload_dir'] This returns a list of file names that were uploaded """ tagBaseDir = tagData('metadata.tag_base_dir', default='') ## # First step is to create a list of directorys that we should make on # the destination cluster. We also want to strip off our local dirs.tag_dir # from the dir names and add teh dstCluster's dirNames = set([os.path.join(dstCluster['config']['dirs.upload_dir'], tagName, makePathRelative(os.path.dirname(f).replace(tagBaseDir, ''))) for f in tagData('files')]) ## # Next we want to take the list of local files, remove the dirs.tag_dir and replace it # with the destination clusters. We maek a list of tuples so we know the local file # and destinatio file name which we will then loop over and upload dstFileNames = [(f, os.path.join(dstCluster['config']['dirs.upload_dir'], tagName, makePathRelative(f.replace(tagBaseDir, '')))) for f in tagData('files')] ## # Next, lets call 'mkdir -p' on all the dirs we need to make on the destination cluster try: makeDirsOnCluster(dstCluster, dirNames) except TryError, err: errorPrint('Caught TryError, ignoring for now: %s - %s ' % (err.msg, str(err.result)))
def checkout(self, options, repo, repoPath, outputPath, branch): fullPath = os.path.join(repo.repoUrl, branch, repoPath) stderr = [] try: # Test to see if it's already checked out, if not continue self._raiseIfCheckout(outputPath) commands.runSingleProgramEx('svn co %s %s' % (fullPath, outputPath), stdoutf=None, stderrf=stderr.append, log=logging.DEBUG) except CheckoutModifiedError: logging.errorPrint('You have uncommited changes to %s, please revert changes or commit them' % outputPath) raise except commands.ProgramRunError: if 'refers to a file, not a directory' in ''.join(stderr): try: tmpPath = os.path.dirname(os.path.join(options('general.codir'), repoPath)) self._raiseIfCheckout(tmpPath) commands.runSystem('rm -rf ' + tmpPath, log=logging.DEBUG) commands.runSystemEx('mkdir -p ' + tmpPath, log=logging.DEBUG) commands.runSingleProgramEx('svn co %s %s' % (os.path.dirname(fullPath), tmpPath), stdoutf=None, stderrf=logging.errorPrintS, log=logging.DEBUG) commands.runSystem('rm -rf ' + outputPath, log=logging.DEBUG) commands.runSystemEx('ln -s %s %s' % (os.path.join(options('general.codir'), repoPath), outputPath), log=logging.DEBUG) except CheckoutModifiedError: logging.errorPrint('You have uncommited changes to %s, please revert changes or commit them' % tmpPath) raise else: for l in stderr: logging.errorPrintS(l) raise logExport(options('general.config_dir'), repo, repoPath, outputPath, branch, CHECKOUT)
try: convertChannel.receive() vmWareDir = 'clovr-vmware.%s' % options('general.version') runSystemEx('mkdir -p ' + vmWareDir, log=True) runSystemEx('mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join(vmWareDir, 'clovr.9-04.x86-64.%s.vmdk' % options('general.version'))) runSystemEx('mkdir -p %s %s' % (os.path.join(vmWareDir, 'keys'), os.path.join(vmWareDir, 'user_data')), log=True) runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir, log=True) fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w') clovrConf = config.configFromMap(dict(version=options('general.version'))) for line in open('/usr/local/projects/clovr/start_clovr.vmx'): fout.write(config.replaceStr(line, clovrConf)) except Exception, err: errorPrint('Converting image failed. Error message:') errorPrint(str(err)) try: amiId = bundleChannel.receive() logPrint('AMI: ' + amiId) except Exception, err: amiId = None errorPrint('Bundling AMI failed for some reason. Error message:') errorPrint(str(err)) if __name__ == '__main__': main(*buildConfigN(OPTIONS))
def runOnElements(conf, query, exc): try: outp = [] runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf ' + query, outp.append, None) hosts = ' '.join(outp) for h in hosts.split(): runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf %s %s' %(exc, h), None, None) except ProgramRunError, err: errorPrint(str(err)) def main(options): conf = configFromStream(open('/tmp/machine.conf')) runOnElements(conf, '-ss', '-ds') runOnElements(conf, '-sql', '-dq') try: runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf -kej ${MY_IP}', None, None) except ProgramRunError, err: errorPrint(str(err)) runOnElements(conf, '-sel', '-de') runOnElements(conf, '-sh', '-dh') try: runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf -dprj global', None, None) except ProgramRunError, err: errorPrint(str(err)) if __name__ == '__main__': main(None)
def installAllStow(): for p in [d for d in os.listdir('/usr/local/stow') if d[0] != '.']: try: installPkg(p) except: logging.errorPrint('Failed to install package: ' + p)
def installAllOptPackages(): for p in [d for d in os.listdir('/opt/opt-packages') if d[0] != '.']: try: installOptPkg(p) except: logging.errorPrint('Failed to install package: ' + p)
def downloadTag(srcCluster, dstCluster, tagName, dstDir=None, baseDir=None): """ srcCluster - Cluster to download the tag from dstCluster - Cluster to download the tag to (this needs to be 'local' for now) tagName - The name of the tag to download dstDir - The destination directory to put downloaded data to. If None, dstDir is assumed to be dstCluster['config']['dirs.upload_dir'] baseDir - When we download a tag we replicate its directory structure, baseDir allows us to remove some portion of the prefix dir in downloading. If baseDir is None then srcCluster['config']['dirs.upload_dir'] is assumed to be the baseDir Neither dstDir or baseDir should consider the 'tagname' as part of their name. This may change in the future though if we want to allow downloading to a new tag name TODO: Consider compressing tags on the remote side before a transfer. It is possible it should be part of the download process. Alternatively it might make more sense for the compression to be part of another process. """ if dstDir is None: dstDir = os.path.join(dstCluster['config']['dirs.upload_dir'], tagName) if baseDir is None: baseDir = srcCluster['config']['dirs.upload_dir'] # # Get the list of files tagData = queryTag('localhost', srcCluster['cluster_name'], [tagName]) # # Some tags have a tag_base_dir metadata element which shows how much of the path # to cut off when transfering. However, it is not guaranteed that every file # in the tag will be in thise base directory, it is meant more as a guide. # If this metadata element exists we want to group the download into 2 steps. # 1) Download all files that exist in the tag_base_dir and cut off tag_base_dir # in the download # 2) The rest baseDirFiles, nonBaseDirFiles = partitionFiles(tagData('files'), baseDir) if baseDirFiles: # # Write those files out to a temporary file so we can use it with --files-from in rysnc tmpFName = os.path.join(dstCluster['config']['general.secure_tmp'], 'rsync-tmp-' + str(time.time())) fout = open(tmpFName, 'w') fout.writelines([f + '\n' for f in baseDirFiles]) fout.close() # # Do the rsync try: makeDirsOnCluster(dstCluster, [dstDir]) except TryError, err: errorPrint('Caught TryError, ignoring for now: %s - %s ' % (err.msg, str(err.result))) rsyncOptions = ' '.join([ dstCluster['config']['rsync.options'], '--files-from=' + tmpFName ]) rsyncFromEx(srcCluster['master']['public_dns'], baseDir, dstDir, rsyncOptions=rsyncOptions, user=srcCluster['config']['rsync.user'], log=True) os.unlink(tmpFName)
def _run(state, batchState): if 'pipeline_name' in batchState: pipelines = yield pipelines_client.pipelineList( 'localhost', 'local', 'guest', batchState['pipeline_name'], detail=True) else: pipelines = [] if not pipelines: _log(batchState, 'First time running, creating pipeline state information') batchState['pipeline_config'] = yield _applyActions( state.innerPipelineConfig(), batchState['actions']) batchState['pipeline_state'] = STARTCLUSTER_STATE # We need to create a fake, local, pipeline for metrics to work batchState['pipeline_name'] = pipeline_misc.checksumInput( batchState['pipeline_config']) batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState[ 'pipeline_name'] batchState['pipeline_config'][ 'pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name'] _log(batchState, 'Pipeline named ' + batchState['pipeline_name']) pipeline = yield pipelines_client.createPipeline( host='localhost', clusterName='local', userName='******', pipelineName=batchState['pipeline_name'], protocol='clovr_wrapper', queue='pipeline.q', config=batchState['pipeline_config'], parentPipeline=state.parentPipeline()) batchState['lgt_wrapper_task_name'] = pipeline['task_name'] _log( batchState, 'Setting number of tasks to 9 (number in a standard lgt_wrapper)') yield _updateTask(batchState, lambda t: t.update(completedTasks=0, numTasks=9)) state.updateBatchState() batchState['state'] = RUNNING_STATE _log(batchState, 'Pipeline started in %s state' % batchState['pipeline_state']) yield _updateTask(batchState, lambda t: t.setState(tasks.task.TASK_RUNNING)) pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs', global_state.make_ref() + '.conf') _log(batchState, 'Creating ergatis configuration') _writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile) if batchState['pipeline_state'] == STARTCLUSTER_STATE: _log(batchState, 'Pipeline is in STARTCLUSTER state') # First see if the cluster exists but is unresponsive try: cluster = yield loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, terminating') terminateTask = yield clusters_client.terminateCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _blockOnTask(terminateTask) except: pass batchState['cluster_task'] = yield startCluster( batchState, 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', int(batchState['pipeline_config']['cluster.EXEC_NODES']), 0, batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'], { 'cluster.MASTER_INSTANCE_TYPE': batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'], 'cluster.MASTER_BID_PRICE': batchState['pipeline_config']['cluster.MASTER_BID_PRICE'], 'cluster.EXEC_INSTANCE_TYPE': batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'], 'cluster.EXEC_BID_PRICE': batchState['pipeline_config']['cluster.EXEC_BID_PRICE'] }) yield _updateTask(batchState, lambda t: t.update(completedTasks=0, numTasks=9)) yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed startcluster').progress()) batchState['pipeline_state'] = REMOTE_LOCAL_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REMOTE_LOCAL_TRANSFER_STATE: _log(batchState, 'Pipeline is in REMOTE_LOCAL_TRANSFER') _log(batchState, 'Making sure cluster is exists in some form') cluster = yield clusters_client.loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, erroring and restarting') raise Exception('Cluster is not responsive') yield state.prerunQueue.addWithDeferred(_remoteLocalTransfer, batchState) yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed remote_local_transfer'). progress()) batchState['pipeline_state'] = DECRYPT_STATE state.updateBatchState() if batchState['pipeline_state'] == DECRYPT_STATE: _log(batchState, 'Pipeline is in DECRYPT') cluster = yield loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') tag = yield tags_client.loadTag( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', _decryptTagName(batchState)) conf = config.configFromStream(open('/tmp/machine.conf')) yield ssh.runProcessSSH(cluster['master']['public_dns'], 'mkdir -p /mnt/lgt_decrypt', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) yield rsync.rsyncTo( cluster['master']['public_dns'], batchState['pipeline_config']['params.DECRYPT_SCRIPT'], '/mnt/', options=conf('rsync.options'), user=conf('rsync.user'), log=True) for f in tag['files']: decryptCmd = ' '.join([ os.path.join( '/mnt', os.path.basename(batchState['pipeline_config'] ['params.DECRYPT_SCRIPT'])), f, '-out-dir', '/mnt/lgt_decrypt', '-remove-encrypted', '-password', batchState['pipeline_config']['params.DECRYPT_PASSWORD'] ]) yield ssh.getOutput(cluster['master']['public_dns'], decryptCmd, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), expected=[0, 253], log=True) tag = yield tags_client.tagData( host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', action='overwrite', tagName=_decryptTagName(batchState), files=['/mnt/lgt_decrypt'], metadata={}, recursive=True, expand=False, compressDir=None) _log( batchState, 'Waiting for tagging of %s to complete - %s' % (_decryptTagName(batchState), tag['task_name'])) yield _blockOnTask( tag['task_name'], cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME']) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed decrypt').progress()) batchState['pipeline_state'] = REFERENCE_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REFERENCE_TRANSFER_STATE: _log(batchState, 'Pipeline is in REFERENCE_TRANSFER state') transfers = [] tags = (batchState['pipeline_config']['input.REF_TAG1'].split(',') + batchState['pipeline_config']['input.REF_TAG2'].split(',')) for tag in tags: tag = tag.strip() output = yield _getOutput(batchState, [ 'vp-transfer-dataset', '-t', '--tag-name=' + tag, '--dst-cluster=' + batchState['pipeline_config']['cluster.CLUSTER_NAME'] ], log=True) transfers.append(output['stdout'].strip()) for task in transfers: yield _blockOnTask(task) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed reference_transfer').progress()) batchState['pipeline_state'] = RUN_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUN_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUN_PIPELINE state') batchState['pipeline_config']['input.INPUT_TAG'] = _decryptTagName( batchState) pipeline = yield pipelines_client.runPipeline( host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', parentPipeline=batchState['pipeline_name'], bareRun=True, queue=state.innerPipelineQueue(), config=batchState['pipeline_config'], overwrite=True) batchState['pipeline_task'] = pipeline['task_name'] yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed run pipeline').progress()) batchState['pipeline_state'] = RUNNING_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUNNING_PIPELINE state') _monitorPipeline(batchState) yield _waitForPipeline(batchState) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed running pipeline').progress()) batchState['pipeline_state'] = HARVEST_STATE state.updateBatchState() if batchState['pipeline_state'] == HARVEST_STATE: _log(batchState, 'Pipeline is in HARVEST state') # Using prerunqueue because we want everything here serialized yield state.prerunQueue.addWithDeferred(_harvestTransfer, batchState) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed harvest').progress()) batchState['pipeline_state'] = SHUTDOWN_STATE state.updateBatchState() if batchState['pipeline_state'] == SHUTDOWN_STATE: _log(batchState, 'Pipeline is in SHUTDOWN state') if 'add_instances_task' in batchState: try: yield _blockOnTask(batchState['add_instances_task'], cluster=batchState['pipeline_config'] ['cluster.CLUSTER_NAME']) except Exception, err: logging.errorPrint(str(err)) log.err(err) yield clusters_client.terminateCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed shutdown').progress()) batchState['pipeline_state'] = COMPLETED_STATE batchState['state'] = COMPLETED_STATE state.updateBatchState()
def main(options, args): logging.DEBUG = options('general.debug') queue = Queue.Queue() ## # Track the downloaded URL names for joining later if specified urls = [] if not args: for line in [l for l in sys.stdin if l.strip()]: md5, url = line.split(' ', 1) url = url.strip() urls.append(url) queue.put((url, md5)) else: for url in args: urls.append(url) queue.put((url, None)) if options('general.join_name') and options('general.join_md5'): md5 = calculateMD5([os.path.join(options('general.base_dir'), options('general.join_name'))]) ## # If they match, then no need to download and exit cleanly if md5 == options('general.join_md5'): return retChans = [threads.runThreadWithChannel(downloadUrls).channel.sendWithChannel((options, queue)) for _ in range(options('general.max_threads'))] successUrls = [] failedUrls = [] for c in retChans: ret = c.receive() while ret is not None: url, succ = ret if succ: successUrls.append(url) else: failedUrls.append(url) ret = c.receive() if failedUrls: for url in failedUrls: logging.errorPrint(url) ## # If any URLs failed, exit with fail sys.exit(1) else: if options('general.join_name'): logging.debugPrint(lambda : 'Joining files into: ' + options('general.join_name')) files = [] for url in urls: files.extend(sorted(getDownloadFilenames(options('general.base_dir'), url))) fout = open(os.path.join(options('general.base_dir'), options('general.join_name')), 'wb') for f in files: logging.debugPrint(lambda : 'Reading: ' + f) fin = open(f, 'rb') d = fin.read(1000000) while d: fout.write(d) d = fin.read(1000000) fin.close() fout.close() logging.debugPrint(lambda : 'Deleting downloaded files after join') for f in files: logging.debugPrint(lambda : 'Deleting: ' + f) deleteFile(f)
def main(options, _args): updateAll = False for o in OPTIONS: if o[0] not in ['vappio_branch', 'clovr_branch' ] and options('general.' + o[0]): break else: updateAll = True clovrBranch = options('general.clovr_branch') vappioBranch = options('general.vappio_branch') try: if options('general.stow') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code', clovrBranch, 'stow', '/usr/local/stow') if options('general.opt_packages') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code', clovrBranch, 'opt-packages', '/opt/opt-packages') if options('general.config_policies') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code', clovrBranch, 'config_policies', '/opt/config_policies') if options('general.vappio_py') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code', vappioBranch, 'vappio-py', '/opt/vappio-py') runSystemEx("""chmod +x /opt/vappio-py/vappio/cli/*.py""") if options('general.vappio_apps') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code', vappioBranch, 'vappio-apps', '/opt/vappio-apps') if options('general.vappio_twisted') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code', vappioBranch, 'vappio-twisted', '/opt/vappio-twisted') if options('general.vappio_scripts') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code', vappioBranch, 'vappio-scripts', '/opt/vappio-scripts') runSystemEx("""chmod -R +x /opt/vappio-scripts""", log=True) runSystemEx("""cp -f /opt/vappio-scripts/clovrEnv.sh /root""", log=True) runSystemEx( """cp -f /opt/vappio-scripts/local /etc/init.d/local""", log=True) runSystemEx( """cp -f /opt/vappio-scripts/rc.local /etc/init.d/rc.local""", log=True) runSystemEx( """cp -f /opt/vappio-scripts/screenrc /root/.screenrc""", log=True) if options('general.clovr_pipelines') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code', clovrBranch, 'clovr_pipelines', '/opt/clovr_pipelines') if options('general.vappio_py_www') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code', vappioBranch, 'vappio-www/py-www', '/var/www/vappio') ## # Only want to do this one when specified # if options('general.vappio_conf'): # grabFromSVN(options, 'https://svn.code.sf.net/p/vappio/code', vappioBranch, 'vappio-conf', '/mnt/vappio-conf') if options('general.hudson') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code', clovrBranch, 'hudson/hudson-config/jobs', '/var/lib/hudson/jobs') grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code', clovrBranch, 'hudson/hudson-scripts', '/opt/hudson') runSystemEx("""chown -R hudson.nogroup /var/lib/hudson/jobs""", log=True) if options('general.clovr_www') or updateAll: grabFromSVN(options, 'https://svn.code.sf.net/p/clovr/code', clovrBranch, 'clovr-www', '/var/www/clovr') except CheckoutModifiedError, err: errorPrint(str(err))
for h in hosts.split(): runSingleProgramEx( conf, '${sge.root}/bin/${sge.arch}/qconf %s %s' % (exc, h), None, None) except ProgramRunError, err: errorPrint(str(err)) def main(options): conf = configFromStream(open('/tmp/machine.conf')) runOnElements(conf, '-ss', '-ds') runOnElements(conf, '-sql', '-dq') try: runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf -kej ${MY_IP}', None, None) except ProgramRunError, err: errorPrint(str(err)) runOnElements(conf, '-sel', '-de') runOnElements(conf, '-sh', '-dh') try: runSingleProgramEx(conf, '${sge.root}/bin/${sge.arch}/qconf -dprj global', None, None) except ProgramRunError, err: errorPrint(str(err)) if __name__ == '__main__': main(None)
def main(options, args): logging.DEBUG = options("general.debug") queue = Queue.Queue() ## # Track the downloaded URL names for joining later if specified urls = [] if not args: for line in [l for l in sys.stdin if l.strip()]: md5, url = line.split(" ", 1) url = url.strip() urls.append(url) queue.put((url, md5)) else: for url in args: urls.append(url) queue.put((url, None)) if options("general.join_name") and options("general.join_md5"): md5 = calculateMD5([os.path.join(options("general.base_dir"), options("general.join_name"))]) ## # If they match, then no need to download and exit cleanly if md5 == options("general.join_md5"): return retChans = [ threads.runThreadWithChannel(downloadUrls).channel.sendWithChannel((options, queue)) for _ in range(options("general.max_threads")) ] successUrls = [] failedUrls = [] for c in retChans: ret = c.receive() while ret is not None: url, succ = ret if succ: successUrls.append(url) else: failedUrls.append(url) ret = c.receive() if failedUrls: for url in failedUrls: logging.errorPrint(url) ## # If any URLs failed, exit with fail sys.exit(1) else: if options("general.join_name"): logging.debugPrint(lambda: "Joining files into: " + options("general.join_name")) files = [] for url in urls: files.extend(sorted(getDownloadFilenames(options("general.base_dir"), url))) fout = open(os.path.join(options("general.base_dir"), options("general.join_name")), "wb") for f in files: logging.debugPrint(lambda: "Reading: " + f) fin = open(f, "rb") d = fin.read(1000000) while d: fout.write(d) d = fin.read(1000000) fin.close() fout.close() logging.debugPrint(lambda: "Deleting downloaded files after join") for f in files: logging.debugPrint(lambda: "Deleting: " + f) deleteFile(f)
runSystemEx('mkdir -p ' + vmWareDir, log=True) runSystemEx( 'mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join( vmWareDir, 'clovr.9-04.x86-64.%s.vmdk' % options('general.version'))) runSystemEx('mkdir -p %s %s' % (os.path.join( vmWareDir, 'keys'), os.path.join(vmWareDir, 'user_data')), log=True) runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir, log=True) fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w') clovrConf = config.configFromMap( dict(version=options('general.version'))) for line in open('/usr/local/projects/clovr/start_clovr.vmx'): fout.write(config.replaceStr(line, clovrConf)) except Exception, err: errorPrint('Converting image failed. Error message:') errorPrint(str(err)) try: amiId = bundleChannel.receive() logPrint('AMI: ' + amiId) except Exception, err: amiId = None errorPrint('Bundling AMI failed for some reason. Error message:') errorPrint(str(err)) if __name__ == '__main__': main(*buildConfigN(OPTIONS))
def _run(state, batchState): if 'pipeline_name' in batchState: pipelines = yield pipelines_client.pipelineList('localhost', 'local', 'guest', batchState['pipeline_name'], detail=True) else: pipelines = [] if not pipelines: _log(batchState, 'First time running, creating pipeline state information') batchState['pipeline_config'] = yield _applyActions(state.innerPipelineConfig(), batchState['actions']) batchState['pipeline_state'] = STARTCLUSTER_STATE # We need to create a fake, local, pipeline for metrics to work batchState['pipeline_name'] = pipeline_misc.checksumInput(batchState['pipeline_config']) batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState['pipeline_name'] batchState['pipeline_config']['pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name'] _log(batchState, 'Pipeline named ' + batchState['pipeline_name']) pipeline = yield pipelines_client.createPipeline(host='localhost', clusterName='local', userName='******', pipelineName=batchState['pipeline_name'], protocol='clovr_wrapper', queue='pipeline.q', config=batchState['pipeline_config'], parentPipeline=state.parentPipeline()) batchState['lgt_wrapper_task_name'] = pipeline['task_name'] _log(batchState, 'Setting number of tasks to 9 (number in a standard lgt_wrapper)') yield _updateTask(batchState, lambda t : t.update(completedTasks=0, numTasks=9)) state.updateBatchState() batchState['state'] = RUNNING_STATE _log(batchState, 'Pipeline started in %s state' % batchState['pipeline_state']) yield _updateTask(batchState, lambda t : t.setState(tasks.task.TASK_RUNNING)) pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs', global_state.make_ref() + '.conf') _log(batchState, 'Creating ergatis configuration') _writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile) if batchState['pipeline_state'] == STARTCLUSTER_STATE: _log(batchState, 'Pipeline is in STARTCLUSTER state') # First see if the cluster exists but is unresponsive try: cluster = yield loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, terminating') terminateTask = yield clusters_client.terminateCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _blockOnTask(terminateTask) except: pass batchState['cluster_task'] = yield startCluster( batchState, 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', int(batchState['pipeline_config']['cluster.EXEC_NODES']), 0, batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'], {'cluster.MASTER_INSTANCE_TYPE': batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'], 'cluster.MASTER_BID_PRICE': batchState['pipeline_config']['cluster.MASTER_BID_PRICE'], 'cluster.EXEC_INSTANCE_TYPE': batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'], 'cluster.EXEC_BID_PRICE': batchState['pipeline_config']['cluster.EXEC_BID_PRICE']}) yield _updateTask(batchState, lambda t : t.update(completedTasks=0, numTasks=9)) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed startcluster' ).progress()) batchState['pipeline_state'] = REMOTE_LOCAL_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REMOTE_LOCAL_TRANSFER_STATE: _log(batchState, 'Pipeline is in REMOTE_LOCAL_TRANSFER') _log(batchState, 'Making sure cluster is exists in some form') cluster = yield clusters_client.loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, erroring and restarting') raise Exception('Cluster is not responsive') yield state.prerunQueue.addWithDeferred(_remoteLocalTransfer, batchState) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed remote_local_transfer' ).progress()) batchState['pipeline_state'] = DECRYPT_STATE state.updateBatchState() if batchState['pipeline_state'] == DECRYPT_STATE: _log(batchState, 'Pipeline is in DECRYPT') cluster = yield loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') tag = yield tags_client.loadTag('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', _decryptTagName(batchState)) conf = config.configFromStream(open('/tmp/machine.conf')) yield ssh.runProcessSSH(cluster['master']['public_dns'], 'mkdir -p /mnt/lgt_decrypt', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) yield rsync.rsyncTo(cluster['master']['public_dns'], batchState['pipeline_config']['params.DECRYPT_SCRIPT'], '/mnt/', options=conf('rsync.options'), user=conf('rsync.user'), log=True) for f in tag['files']: decryptCmd = ' '.join([os.path.join('/mnt', os.path.basename(batchState['pipeline_config']['params.DECRYPT_SCRIPT'])), f, '-out-dir', '/mnt/lgt_decrypt', '-remove-encrypted', '-password', batchState['pipeline_config']['params.DECRYPT_PASSWORD']]) yield ssh.getOutput(cluster['master']['public_dns'], decryptCmd, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), expected=[0, 253], log=True) tag = yield tags_client.tagData(host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', action='overwrite', tagName=_decryptTagName(batchState), files=['/mnt/lgt_decrypt'], metadata={}, recursive=True, expand=False, compressDir=None) _log(batchState, 'Waiting for tagging of %s to complete - %s' % (_decryptTagName(batchState), tag['task_name'])) yield _blockOnTask(tag['task_name'], cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME']) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed decrypt' ).progress()) batchState['pipeline_state'] = REFERENCE_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REFERENCE_TRANSFER_STATE: _log(batchState, 'Pipeline is in REFERENCE_TRANSFER state') transfers = [] tags = (batchState['pipeline_config']['input.REF_TAG1'].split(',') + batchState['pipeline_config']['input.REF_TAG2'].split(',')) for tag in tags: tag = tag.strip() output = yield _getOutput(batchState, ['vp-transfer-dataset', '-t', '--tag-name=' + tag, '--dst-cluster=' + batchState['pipeline_config']['cluster.CLUSTER_NAME']], log=True) transfers.append(output['stdout'].strip()) for task in transfers: yield _blockOnTask(task) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed reference_transfer' ).progress()) batchState['pipeline_state'] = RUN_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUN_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUN_PIPELINE state') batchState['pipeline_config']['input.INPUT_TAG'] = _decryptTagName(batchState) pipeline = yield pipelines_client.runPipeline(host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', parentPipeline=batchState['pipeline_name'], bareRun=True, queue=state.innerPipelineQueue(), config=batchState['pipeline_config'], overwrite=True) batchState['pipeline_task'] = pipeline['task_name'] yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed run pipeline' ).progress()) batchState['pipeline_state'] = RUNNING_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUNNING_PIPELINE state') _monitorPipeline(batchState) yield _waitForPipeline(batchState) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed running pipeline' ).progress()) batchState['pipeline_state'] = HARVEST_STATE state.updateBatchState() if batchState['pipeline_state'] == HARVEST_STATE: _log(batchState, 'Pipeline is in HARVEST state') # Using prerunqueue because we want everything here serialized yield state.prerunQueue.addWithDeferred(_harvestTransfer, batchState) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed harvest' ).progress()) batchState['pipeline_state'] = SHUTDOWN_STATE state.updateBatchState() if batchState['pipeline_state'] == SHUTDOWN_STATE: _log(batchState, 'Pipeline is in SHUTDOWN state') if 'add_instances_task' in batchState: try: yield _blockOnTask(batchState['add_instances_task'], cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME']) except Exception, err: logging.errorPrint(str(err)) log.err(err) yield clusters_client.terminateCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed shutdown' ).progress()) batchState['pipeline_state'] = COMPLETED_STATE batchState['state'] = COMPLETED_STATE state.updateBatchState()
## # Now, copy up all of the files for l, d in dstFileNames: scpToEx(dstCluster['master']['public_dns'], l, d, user=srcCluster['config']['ssh.user'], options=srcCluster['config']['ssh.options'], log=True) ## # We are uploading as root, so chown everything to the user that everything in vappio will be done under try: runSystemInstanceEx(dstCluster['master'], 'chown %s %s' % (dstCluster['config']['vappio.user'], d), None, errorPrintS, user=dstCluster['config']['ssh.user'], options=dstCluster['config']['ssh.options'], log=True) except ProgramRunError: errorPrint('Chown failed on ' + d) ## # return the list of uploaded filenames return [d for l, d in dstFileNames] def partitionFiles(files, baseDir): if baseDir: baseDirFiles = [f.replace(baseDir, '') for f in files if f.startswith(baseDir)] downloadFiles = [f for f in files if not f.startswith(baseDir)] return (baseDirFiles, downloadFiles)
def downloadTag(srcCluster, dstCluster, tagName, dstDir=None, baseDir=None): """ srcCluster - Cluster to download the tag from dstCluster - Cluster to download the tag to (this needs to be 'local' for now) tagName - The name of the tag to download dstDir - The destination directory to put downloaded data to. If None, dstDir is assumed to be dstCluster['config']['dirs.upload_dir'] baseDir - When we download a tag we replicate its directory structure, baseDir allows us to remove some portion of the prefix dir in downloading. If baseDir is None then srcCluster['config']['dirs.upload_dir'] is assumed to be the baseDir Neither dstDir or baseDir should consider the 'tagname' as part of their name. This may change in the future though if we want to allow downloading to a new tag name TODO: Consider compressing tags on the remote side before a transfer. It is possible it should be part of the download process. Alternatively it might make more sense for the compression to be part of another process. """ if dstDir is None: dstDir = os.path.join(dstCluster['config']['dirs.upload_dir'], tagName) if baseDir is None: baseDir = srcCluster['config']['dirs.upload_dir'] # # Get the list of files tagData = queryTag('localhost', srcCluster['cluster_name'], [tagName]) # # Some tags have a tag_base_dir metadata element which shows how much of the path # to cut off when transfering. However, it is not guaranteed that every file # in the tag will be in thise base directory, it is meant more as a guide. # If this metadata element exists we want to group the download into 2 steps. # 1) Download all files that exist in the tag_base_dir and cut off tag_base_dir # in the download # 2) The rest baseDirFiles, nonBaseDirFiles = partitionFiles(tagData('files'), baseDir) if baseDirFiles: # # Write those files out to a temporary file so we can use it with --files-from in rysnc tmpFName = os.path.join(dstCluster['config']['general.secure_tmp'], 'rsync-tmp-' + str(time.time())) fout = open(tmpFName, 'w') fout.writelines([f + '\n' for f in baseDirFiles]) fout.close() # # Do the rsync try: makeDirsOnCluster(dstCluster, [dstDir]) except TryError, err: errorPrint('Caught TryError, ignoring for now: %s - %s ' % (err.msg, str(err.result))) rsyncOptions = ' '.join([dstCluster['config']['rsync.options'], '--files-from=' + tmpFName]) rsyncFromEx(srcCluster['master']['public_dns'], baseDir, dstDir, rsyncOptions=rsyncOptions, user=srcCluster['config']['rsync.user'], log=True) os.unlink(tmpFName)
user=srcCluster['config']['ssh.user'], options=srcCluster['config']['ssh.options'], log=True) ## # We are uploading as root, so chown everything to the user that everything in vappio will be done under try: runSystemInstanceEx(dstCluster['master'], 'chown %s %s' % (dstCluster['config']['vappio.user'], d), None, errorPrintS, user=dstCluster['config']['ssh.user'], options=dstCluster['config']['ssh.options'], log=True) except ProgramRunError: errorPrint('Chown failed on ' + d) ## # return the list of uploaded filenames return [d for l, d in dstFileNames] def partitionFiles(files, baseDir): if baseDir: baseDirFiles = [ f.replace(baseDir, '') for f in files if f.startswith(baseDir) ] downloadFiles = [f for f in files if not f.startswith(baseDir)] return (baseDirFiles, downloadFiles) else: return ([], files)