def run(options): logging.logPrint('Starting') batchConfig = config.configFromStream(open(options.configFile), lazy=True) machineConf = config.configFromStream(open('/tmp/machine.conf')) state = State(options.workflowConfig, options.batchStatesFile, _validateWrapper(batchConfig('batch_pipeline.pipeline.PIPELINE_TEMPLATE'), pipeline_misc.determineWrapper(machineConf, batchConfig('batch_pipeline.pipeline.PIPELINE_TEMPLATE'))), _interpretBatchFile(options.batchFile), _extractInnerPipelineConfig(batchConfig), batchConfig('pipeline.PIPELINE_WRAPPER_NAME'), int(batchConfig('batch.options.CONCURRENT_PRERUN')), int(batchConfig('batch.options.CONCURRENT_PIPELINES')), int(batchConfig('batch.options.CONCURRENT_POSTRUN'))) logging.logPrint('Queuing any incomplete work') queueCount = _queueIncompleteWork(state) logging.logPrint('Queued: %d' % queueCount) if state.pipelinesQueue.hasWork(): yield defer_work_queue.waitForCompletion(state.pipelinesQueue) for batchState in state.batchStates.values(): if 'state' not in batchState or batchState['state'] == 'failed': raise JobFailed()
def confIfPipelineConfigSet(conf, options): """ Takes a conf, checks to see if a pipeline conf file is specified, if so it loads it up and applies it OVER any options specified on the command line. This may seem counter intuitive but it makes other things easier, for example a pipeline redefining anything in the machines.conf since that is also in this conf. It then applies the functions in the OPTIONS variable in the values in the config file """ if conf('CONFIG_FILE', default=None) is not None: fconf = config.configFromStream(open(conf('CONFIG_FILE'))) keys = fconf.keys() m = {} for o in options: ## # Get the name of the option, it's the first element of the tuple name = o[0] f = o[4] if name in keys: m[name] = applyIfCallable(f(fconf(name)), conf) ## # lazy=True is for saftey incase there is a value in the CONFIG_FILE that we use that # really depends on a value in the map we just created return config.configFromMap( m, config.configFromStream(open(conf('CONFIG_FILE')), conf, lazy=False)) else: return conf
def confIfPipelineConfigSet(conf, options): """ Takes a conf, checks to see if a pipeline conf file is specified, if so it loads it up and applies it OVER any options specified on the command line. This may seem counter intuitive but it makes other things easier, for example a pipeline redefining anything in the machines.conf since that is also in this conf. It then applies the functions in the OPTIONS variable in the values in the config file """ if conf("CONFIG_FILE", default=None) is not None: fconf = config.configFromStream(open(conf("CONFIG_FILE"))) keys = fconf.keys() m = {} for o in options: ## # Get the name of the option, it's the first element of the tuple name = o[0] f = o[4] if name in keys: m[name] = applyIfCallable(f(fconf(name)), conf) ## # lazy=True is for saftey incase there is a value in the CONFIG_FILE that we use that # really depends on a value in the map we just created return config.configFromMap(m, config.configFromStream(open(conf("CONFIG_FILE")), conf, lazy=False)) else: return conf
def run(options): logging.logPrint('Starting') batchConfig = config.configFromStream(open(options.configFile), lazy=True) machineConf = config.configFromStream(open('/tmp/machine.conf')) state = State( options.workflowConfig, options.batchStatesFile, _validateWrapper( batchConfig('batch_pipeline.pipeline.PIPELINE_TEMPLATE'), pipeline_misc.determineWrapper( machineConf, batchConfig('batch_pipeline.pipeline.PIPELINE_TEMPLATE'))), _interpretBatchFile(options.batchFile), _extractInnerPipelineConfig(batchConfig), batchConfig('pipeline.PIPELINE_WRAPPER_NAME'), int(batchConfig('batch.options.CONCURRENT_PRERUN')), int(batchConfig('batch.options.CONCURRENT_PIPELINES')), int(batchConfig('batch.options.CONCURRENT_POSTRUN'))) logging.logPrint('Queuing any incomplete work') queueCount = _queueIncompleteWork(state) logging.logPrint('Queued: %d' % queueCount) if state.pipelinesQueue.hasWork(): yield defer_work_queue.waitForCompletion(state.pipelinesQueue) for batchState in state.batchStates.values(): if 'state' not in batchState or batchState['state'] == 'failed': raise JobFailed()
def main(options, args): if not options('general.pipeline_resume'): conf = config.configFromStream(open(options('general.pipeline_config')), lazy=True) ret = pipeline.validatePipelineConfig(options('general.host'), options('general.cluster'), options('general.bare_run'), conf) if ret['errors']: for e in ret['errors']: print '\t'.join(['ERROR', ','.join(e['keys']), e['message']]) # Exit with an error sys.exit(1) if not options('general.validate'): p = pipeline.runPipeline(options('general.host'), options('general.cluster'), options('general.pipeline_parent'), options('general.bare_run'), conf, options('general.pipeline_queue'), options('general.overwrite')) if options('general.print_task_name'): print p['task_name'] else: runTaskStatus(p['task_name']) else: p = pipeline.resumePipeline(options('general.host'), options('general.cluster'), options('general.pipeline_resume')) if options('general.print_task_name'): print p['task_name'] else: runTaskStatus(p['task_name'])
def _delayAutoshutdown(state, batchState): _log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file') try: clusters = yield clusters_client.listClusters('localhost', {'cluster_name': batchState['pipeline_config']['cluster.CLUSTER_NAME']}, 'guest') cluster = clusters[0] if batchState.get('state', None) == COMPLETED_STATE: _log(batchState, 'AUTOSHUTDOWN: Pipeline complete, done') if batchState.get('state', None) != RUNNING_STATE: _log(batchState, 'AUTOSHUTDOWN: Pipeline not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) elif cluster['state'] == 'running': # We need the SSH options from the machine.conf, ugh I hate these OOB dependencies conf = config.configFromStream(open('/tmp/machine.conf')) _log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown') yield ssh.runProcessSSH(cluster['master']['public_dns'], 'touch /var/vappio/runtime/delayautoshutdown', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) _log(batchState, 'AUTOSHUTDOWN: Setting up next call') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) else: _log(batchState, 'AUTOSHUTDOWN: Cluster not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) except: _log(batchState, 'AUTOSHUTDOWN: Cluster does not exist, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
def instantiateCredential(conf, cred): """ Takes a credential and instanitates it. It returns a Record that has all of the information users of that instantiated credential will need """ if not conf('config_loaded', default=False): conf = config.configFromConfig( conf, base=config.configFromStream( open(conf('conf_file', default=DEFAULT_CONFIG_FILE)), base=conf)) certFile = os.path.join( conf('general.secure_tmp'), cred.name + '_cert.pem') keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem') if not os.path.exists(certFile) or open(certFile).read() != cred.cert: open(certFile, 'w').write(cred.cert) if not os.path.exists(keyFile) or open(keyFile).read() != cred.pkey: open(keyFile, 'w').write(cred.pkey) newCred = functional.Record( name=cred.name, conf=conf, cert=certFile, pkey=keyFile, ec2URL=None, env={}) if 'ec2_url' in cred.metadata: newCred = newCred.update( env=functional.updateDict(newCred.env, dict(EC2_URL=cred.metadata['ec2_url']))) yield _createGroups(newCred) yield _createKeypair(newCred) defer.returnValue(newCred)
def instantiateCredential(conf, cred): """ Takes a credential and instanitates it. It returns a Record that has all of the information users of that instantiated credential will need """ if not conf('config_loaded', default=False): conf = config.configFromMap({'config_loaded': True}, base=config.configFromStream(open( conf('general.conf_file', default=DEFAULT_CONFIG_FILE)), base=conf)) certFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert.pem') keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem') if not os.path.exists(certFile) or open(certFile).read() != cred.cert: open(certFile, 'w').write(cred.cert) if not os.path.exists(keyFile) or open(keyFile).read() != cred.pkey: open(keyFile, 'w').write(cred.pkey) newCred = functional.Record(cert=certFile, pkey=keyFile, ec2URL=None, env={}) if 'ec2_url' in cred.metadata: return (conf, newCred.update(env=functional.updateDict( newCred.env, dict(EC2_URL=cred.metadata['ec2_url'])))) else: return (conf, newCred)
def _setQueue(taskName, batchState): yield _blockOnTask(taskName) cluster = yield loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield defer_utils.tryUntil( 10, lambda: _getOutput(batchState, [ '/opt/clovr_pipelines/workflow/project_saved_templates/clovr_lgt_wrapper/set_queue.sh', cluster['master']['public_dns'] ], log=True), onFailure=defer_utils.sleep(2)) conf = config.configFromStream(open('/tmp/machine.conf')) # Remove autoshutdown, we want no part of that yield ssh.runProcessSSH(cluster['master']['public_dns'], 'rm -v /var/vappio/runtime/noautoshutdown', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True)
def _credential(): if os.path.exists('/tmp/cred-info'): cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split( '\t') return { 'name': 'local', 'desc': 'Local credential', 'ctype': ctype, 'cert': open(cert).read(), 'pkey': open(pkey).read(), 'metadata': metadata and dict([v.split('=', 1) for v in metadata.split(',')]) or {}, 'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True) } else: return { 'name': 'local', 'desc': 'Local credential', 'ctype': 'local', 'cert': None, 'pkey': None, 'metadata': {}, 'conf': config.configFromMap({}) }
def __init__(self, conf): self.conf = conf self.persistManager = persist.ClusterPersistManager() self.machineConf = config.configFromStream(open(conf('config.machine_conf')), base=config.configFromEnv()) self.clusterLocks = lock_manager.LockManager() self.unresponsiveClusters = {}
def _loadTag(): tagPath = _createTagPath(self.conf, tagName) if not os.path.exists(tagPath) and not os.path.exists(tagPath + '.phantom'): return defer.fail(TagNotFoundError(tagName)) if os.path.exists(tagPath + '.phantom'): phantom = config.configFromStream(open(tagPath + '.phantom'), lazy=True) else: phantom = None if os.path.exists(tagPath + '.metadata'): metadata = json.loads(open(tagPath + '.metadata').read()) else: metadata = {} if os.path.exists(tagPath): files = [ f.rstrip('\n') for f in open(tagPath).readlines() if f.strip() ] else: files = [] return Tag(tagName=tagName, files=files, metadata=metadata, phantom=phantom, taskName=metadata.get('task_name'))
def __init__(self, conf): self.conf = conf self.persistManager = persist.ClusterPersistManager() self.machineConf = config.configFromStream(open( conf('config.machine_conf')), base=config.configFromEnv()) self.clusterLocks = lock_manager.LockManager() self.unresponsiveClusters = {}
def instantiateCredential(conf, cred): if not conf('config_loaded', default=False): conf = config.configFromMap({'config_loaded': True}, base=config.configFromStream(open( conf('general.conf_file', default=DEFAULT_CONFIG_FILE)), base=conf)) return (conf, None)
def __init__(self, conf): self.conf = conf self.pipelinePersist = persist.PipelinePersistManager() self.tagNotify = tag_notify_listener.TagNotifyListener() self.machineconf = config.configFromStream(open(conf('config.machine_conf')), base=config.configFromEnv()) self.pipelinesCache = pipelines_cache.PipelinesCache(self.machineconf, self.pipelinePersist, self.tagNotify) self.pipelinesMonitor = pipeline_monitor.PipelineMonitorManager()
def loadTagFile(fname): """ Loads a tagfile, returns a config object of attributes Also considering a .phantom type which would represent files that don't really exist. I think this makes sense as you should be able to tarnsfer .phantom files around but .metadata's should be generated when you make a tag Will explain more abou this in a wiki page somewhere... """ ## # Phantom filse are in a format that configFromStream can read. This is because phantom files # are expected to be written and modified by humans. .metadata files on the other hand # are just expected to be the produce of a machine storing information so uses json if os.path.exists(fname + '.phantom'): ## # Put everythin under phantom # We want to do it lazily too since we will be adding # data it can access later phantom = configFromMap( { 'phantom_tag': True, 'phantom': configToDict( configFromStream(open(fname + '.phantom'), lazy=True)) }, lazy=True) else: phantom = configFromMap({}) ## # If the fname actually exists, open its meta data + files # if the fname does not exist but the phantom does, return the phantom # otherwise, throw an exception about missing the tagfile if os.path.exists(fname): if os.path.exists(fname + '.metadata'): metadata = configFromMap( {'metadata': json.loads(open(fname + '.metadata').read())}, phantom, lazy=True) else: metadata = configFromMap({}, phantom) return configFromMap( {'files': [f.strip() for f in open(fname) if f.strip()]}, metadata, lazy=True) elif not os.path.exists(fname) and os.path.exists(fname + '.phantom'): if os.path.exists(fname + '.metadata'): metadata = configFromMap( {'metadata': json.loads(open(fname + '.metadata').read())}, phantom, lazy=True) return metadata else: return phantom else: raise MissingTagFileError(fname)
def __init__(self, conf): self.conf = conf self.pipelinePersist = persist.PipelinePersistManager() self.tagNotify = tag_notify_listener.TagNotifyListener() self.machineconf = config.configFromStream(open( conf('config.machine_conf')), base=config.configFromEnv()) self.pipelinesCache = pipelines_cache.PipelinesCache( self.machineconf, self.pipelinePersist, self.tagNotify) self.pipelinesMonitor = pipeline_monitor.PipelineMonitorManager()
def createExecDataFile(conf, master, masterMachineConf): """ Creates a exec data file as the perl start_cluster works This is very similar to createMasterDataFile, should be refactored a bit """ outName = os.path.join('/tmp', str(time.time())) ## # Going to load the master machine.conf and modify node type masterConf = config.configFromStream(open(masterMachineConf), lazy=True) masterConf = config.configFromMap({'NODE_TYPE': EXEC_NODE}, masterConf, lazy=True) fout = open(outName, 'w') fout.write('\n'.join([ k + '=' + str(v) for k, v in config.configToDict(masterConf).iteritems() ])) fout.close() template = open(conf('cluster.exec_user_data_tmpl')).read() clusterPrivateKey = open(conf('cluster.cluster_private_key')).read() outf = [] runSingleProgramEx('ssh-keygen -y -f ' + conf('cluster.cluster_private_key'), outf.append, None, log=True) if conf('general.ctype') == 'ec2': template = template.replace('<TMPL_VAR NAME=MASTER_DNS>', master['private_dns']) else: template = template.replace('<TMPL_VAR NAME=MASTER_DNS>', master['public_dns']) clusterPublicKey = ''.join(outf) template = template.replace('<TMPL_VAR NAME=CLUSTER_PRIVATE_KEY>', clusterPrivateKey) template = template.replace('<TMPL_VAR NAME=CLUSTER_PUBLIC_KEY>', clusterPublicKey) template = template.replace('<TMPL_VAR NAME=MACHINE_CONF>', open(outName).read().replace('${', '\\${')) os.remove(outName) outf = os.path.join(conf('general.secure_tmp'), 'exec_user_data.sh') open(outf, 'w').write(template) return outf
def createCluster(request): persistManager = request.state.persistManager baseConf = config.configFromStream(open('/tmp/machine.conf')) cluster = persist.Cluster( request.body['cluster_name'], request.body['user_name'], request.body['cred_name'], config.configFromMap(request.body['conf'], base=baseConf)) yield persistManager.saveCluster(cluster) defer.returnValue(request)
def createCluster(request): persistManager = request.state.persistManager baseConf = config.configFromStream(open('/tmp/machine.conf')) cluster = persist.Cluster(request.body['cluster_name'], request.body['user_name'], request.body['cred_name'], config.configFromMap(request.body['conf'], base=baseConf)) yield persistManager.saveCluster(cluster) defer.returnValue(request)
def instantiateCredential(conf, cred): if not conf('config_loaded', default=False): conf = config.configFromMap({'config_loaded': True}, base=config.configFromStream(open( conf('general.conf_file')), base=conf)) certFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert.pem') keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem') if not os.path.exists(certFile) and not os.path.exists(keyFile): tmpCertFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert-tmp.pem') tmpKeyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key-tmp.pem') if 'ec2_url' not in cred.metadata: raise Exception('You must have an ec2_url') parsedUrl = urlparse.urlparse(cred.metadata['ec2_url']) if ':' not in parsedUrl.netloc: raise Exception('Your URL must contain a port') host, port = parsedUrl.netloc.split(':') fout = open(tmpCertFile, 'w') fout.write(cred.cert) fout.close() fout = open(tmpKeyFile, 'w') fout.write(cred.pkey) fout.close() try: commands.runSystemEx(' '.join([ 'nimbusCerts2EC2.py', '--in-cert=' + tmpCertFile, '--out-cert=' + certFile, '--in-key=' + tmpKeyFile, '--out-key=' + keyFile, '--java-cert-dir=/tmp', '--java-cert-host=' + host, '--java-cert-port=' + port ]) + ' > /dev/null 2>&1', log=True) commands.runSystemEx('chmod +r ' + keyFile) finally: os.unlink(tmpCertFile) os.unlink(tmpKeyFile) ec2Home = '/opt/ec2-api-tools-1.3-42584' newCred = func.Record( cert=certFile, pkey=keyFile, ec2Path=os.path.join(ec2Home, 'bin'), env=dict(EC2_JVM_ARGS='-Djavax.net.ssl.trustStore=/tmp/jssecacerts', EC2_HOME=ec2Home, EC2_URL=cred.metadata['ec2_url'])) if os.path.exists(conf('cluster.cluster_private_key') + '.pub'): pubKey = open(conf('cluster.cluster_private_key') + '.pub').read().rstrip() ec2_control.addKeypair(newCred, '"' + conf('cluster.key') + '||' + pubKey + '"') return (conf, newCred)
def runPipelineConfig(taskName, name, pipeline, conf, queue=None): """ Takes a config object representing a pipeline options, validates those options in pipeline.OPTIONS and passes the results onto runPipelineWithConfig """ ## # Mocheezmo way to have it load a conf file. This will be removed in the future tmpConfigName = os.path.join('/tmp', str(time.time()) + '.config') options = list(pipeline.OPTIONS) options.append( ('conf', '', '--conf', 'Conf file (DO NOT SPECIFY, FOR INTERNAL USE)', const('/tmp/machine.conf'))) options.append(( 'CONFIG_FILE', '-c', '--CONFIG_FILE', 'Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line', const(tmpConfigName))) ## # Load up machine.conf and apply it to our current config conf = config.configFromConfig(conf, config.configFromStream( open('/tmp/machine.conf'), config.configFromEnv()), lazy=True) vals = {} for o in options: vals[o[0]] = cli.applyOption(conf(o[0], default=None), o, conf) conf = config.configFromMap(vals, conf) ## # For some ergatis trickery we then need to output this config to a temp file so ergatis can pull variables from it confDict = config.configToDict(conf) confVals = {} cv = [('.'.join(k.split('.')[:-1]), k.split('.')[-1], v) for k, v in confDict.iteritems()] for s, k, v in cv: confVals.setdefault(s, {})[k] = v fout = open(tmpConfigName, 'w') for s, d in confVals.iteritems(): if s not in ['', 'env']: fout.write('[' + s + ']\n') for k, v in d.iteritems(): fout.write('%s=%s\n' % (k, str(v))) fout.close() return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
def runPipelineConfig(taskName, name, pipeline, conf, queue=None): """ Takes a config object representing a pipeline options, validates those options in pipeline.OPTIONS and passes the results onto runPipelineWithConfig """ ## # Mocheezmo way to have it load a conf file. This will be removed in the future tmpConfigName = os.path.join("/tmp", str(time.time()) + ".config") options = list(pipeline.OPTIONS) options.append(("conf", "", "--conf", "Conf file (DO NOT SPECIFY, FOR INTERNAL USE)", const("/tmp/machine.conf"))) options.append( ( "CONFIG_FILE", "-c", "--CONFIG_FILE", "Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line", const(tmpConfigName), ) ) ## # Load up machine.conf and apply it to our current config conf = config.configFromConfig( conf, config.configFromStream(open("/tmp/machine.conf"), config.configFromEnv()), lazy=True ) vals = {} for o in options: vals[o[0]] = cli.applyOption(conf(o[0], default=None), o, conf) conf = config.configFromMap(vals, conf) ## # For some ergatis trickery we then need to output this config to a temp file so ergatis can pull variables from it confDict = config.configToDict(conf) confVals = {} cv = [(".".join(k.split(".")[:-1]), k.split(".")[-1], v) for k, v in confDict.iteritems()] for s, k, v in cv: confVals.setdefault(s, {})[k] = v fout = open(tmpConfigName, "w") for s, d in confVals.iteritems(): if s not in ["", "env"]: fout.write("[" + s + "]\n") for k, v in d.iteritems(): fout.write("%s=%s\n" % (k, str(v))) fout.close() return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
def createCluster(request): """Instantiates a skeleton of the cluster that will be imported. This cluster will later have the proper attributes and values populated from the source cluster. """ persistManager = request.state.persistManager baseConf = config.configFromStream(open('/tmp/machine.conf')) cluster = persist.Cluster(request.body['dst_cluster'], request.body['user_name'], request.body['cred_name'], config.configFromMap({'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'}, base=baseConf)) yield persistManager.saveCluster(cluster) defer.returnValue(request)
def createExecDataFile(conf, master, masterMachineConf): """ Creates a exec data file as the perl start_cluster works This is very similar to createMasterDataFile, should be refactored a bit """ outName = os.path.join('/tmp', str(time.time())) ## # Going to load the master machine.conf and modify node type masterConf = config.configFromStream(open(masterMachineConf), lazy=True) masterConf = config.configFromMap({'NODE_TYPE': EXEC_NODE}, masterConf, lazy=True) fout = open(outName, 'w') fout.write('\n'.join([k + '=' + str(v) for k, v in config.configToDict(masterConf).iteritems()])) fout.close() template = open(conf('cluster.exec_user_data_tmpl')).read() clusterPrivateKey = open(conf('cluster.cluster_private_key')).read() outf = [] runSingleProgramEx('ssh-keygen -y -f ' + conf('cluster.cluster_private_key'), outf.append, None, log=True) if conf('general.ctype') == 'ec2': template = template.replace('<TMPL_VAR NAME=MASTER_DNS>', master['private_dns']) else: template = template.replace('<TMPL_VAR NAME=MASTER_DNS>', master['public_dns']) clusterPublicKey = ''.join(outf) template = template.replace('<TMPL_VAR NAME=CLUSTER_PRIVATE_KEY>', clusterPrivateKey) template = template.replace('<TMPL_VAR NAME=CLUSTER_PUBLIC_KEY>', clusterPublicKey) template = template.replace('<TMPL_VAR NAME=MACHINE_CONF>', open(outName).read().replace('${', '\\${')) os.remove(outName) outf = os.path.join(conf('general.secure_tmp'), 'exec_user_data.sh') open(outf, 'w').write(template) return outf
def createCluster(request): """Instantiates a skeleton of the cluster that will be imported. This cluster will later have the proper attributes and values populated from the source cluster. """ persistManager = request.state.persistManager baseConf = config.configFromStream(open('/tmp/machine.conf')) cluster = persist.Cluster( request.body['dst_cluster'], request.body['user_name'], request.body['cred_name'], config.configFromMap( {'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'}, base=baseConf)) yield persistManager.saveCluster(cluster) defer.returnValue(request)
def _credential(): if os.path.exists('/tmp/cred-info'): cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split('\t') return {'name': 'local', 'desc': 'Local credential', 'ctype': ctype, 'cert': open(cert).read(), 'pkey': open(pkey).read(), 'metadata': metadata and dict([v.split('=', 1) for v in metadata.split(',')]) or {}, 'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True)} else: return {'name': 'local', 'desc': 'Local credential', 'ctype': 'local', 'cert': None, 'pkey': None, 'metadata': {}, 'conf': config.configFromMap({})}
def instantiateCredential(conf, cred): if not conf('config_loaded', default=False): conf = config.configFromMap({'config_loaded': True}, base=config.configFromStream(open(conf('general.conf_file')), base=conf)) certFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert.pem') keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem') if not os.path.exists(certFile) and not os.path.exists(keyFile): tmpCertFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert-tmp.pem') tmpKeyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key-tmp.pem') if 'ec2_url' not in cred.metadata: raise Exception('You must have an ec2_url') parsedUrl = urlparse.urlparse(cred.metadata['ec2_url']) if ':' not in parsedUrl.netloc: raise Exception('Your URL must contain a port') host, port = parsedUrl.netloc.split(':') fout = open(tmpCertFile, 'w') fout.write(cred.cert) fout.close() fout = open(tmpKeyFile, 'w') fout.write(cred.pkey) fout.close() try: commands.runSystemEx(' '.join(['nimbusCerts2EC2.py', '--in-cert=' + tmpCertFile, '--out-cert=' + certFile, '--in-key=' + tmpKeyFile, '--out-key=' + keyFile, '--java-cert-dir=/tmp', '--java-cert-host=' + host, '--java-cert-port=' + port]) + ' > /dev/null 2>&1', log=True) commands.runSystemEx('chmod +r ' + keyFile) finally: os.unlink(tmpCertFile) os.unlink(tmpKeyFile) ec2Home = '/opt/ec2-api-tools-1.3-42584' newCred = func.Record(cert=certFile, pkey=keyFile, ec2Path=os.path.join(ec2Home, 'bin'), env=dict(EC2_JVM_ARGS='-Djavax.net.ssl.trustStore=/tmp/jssecacerts', EC2_HOME=ec2Home, EC2_URL=cred.metadata['ec2_url'])) if os.path.exists(conf('cluster.cluster_private_key') + '.pub'): pubKey = open(conf('cluster.cluster_private_key') + '.pub').read().rstrip() ec2_control.addKeypair(newCred, '"' + conf('cluster.key') + '||' + pubKey + '"') return (conf, newCred)
def _delayAutoshutdown(state, batchState): _log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file') try: clusters = yield clusters_client.listClusters('localhost', { 'cluster_name': batchState['pipeline_config']['cluster.CLUSTER_NAME'] }, 'guest') cluster = clusters[0] if batchState.get('state', None) == COMPLETED_STATE: _log(batchState, 'AUTOSHUTDOWN: Pipeline complete, done') if batchState.get('state', None) != RUNNING_STATE: _log(batchState, 'AUTOSHUTDOWN: Pipeline not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) elif cluster['state'] == 'running': # We need the SSH options from the machine.conf, ugh I hate these OOB dependencies conf = config.configFromStream(open('/tmp/machine.conf')) _log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown') yield ssh.runProcessSSH( cluster['master']['public_dns'], 'touch /var/vappio/runtime/delayautoshutdown', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) _log(batchState, 'AUTOSHUTDOWN: Setting up next call') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) else: _log(batchState, 'AUTOSHUTDOWN: Cluster not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) except: _log(batchState, 'AUTOSHUTDOWN: Cluster does not exist, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
def _delayAutoshutdown(state, batchState): _log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file') if (('cluster_task' not in batchState or batchState.get('state', None) != RUNNING_STATE) and batchState.get('state', None) != COMPLETED_STATE): # Not ready yet _log(batchState, 'AUTOSHUTDOWN: No cluster or not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) elif ('cluster_task' in batchState and batchState.get('state', None) == RUNNING_STATE and batchState.get('pipeline_state', None) != SHUTDOWN_STATE): # Ready to see if resizing _log(batchState, 'AUTOSHUTDOWN: Making sure cluster is up') yield _blockOnTask(batchState['cluster_task']) try: cluster = yield loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') # We need the SSH options from the machine.conf, ugh I hate these OOB dependencies conf = config.configFromStream(open('/tmp/machine.conf')) _log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown') yield ssh.runProcessSSH( cluster['master']['public_dns'], 'touch /var/vappio/runtime/delayautoshutdown', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) except: pass _log(batchState, 'AUTOSHUTDOWN: Setting up next call') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) else: _log(batchState, 'AUTOSHUTDOWN: Giving up on this')
def loadTagFile(fname): """ Loads a tagfile, returns a config object of attributes Also considering a .phantom type which would represent files that don't really exist. I think this makes sense as you should be able to tarnsfer .phantom files around but .metadata's should be generated when you make a tag Will explain more abou this in a wiki page somewhere... """ ## # Phantom filse are in a format that configFromStream can read. This is because phantom files # are expected to be written and modified by humans. .metadata files on the other hand # are just expected to be the produce of a machine storing information so uses json if os.path.exists(fname + '.phantom'): ## # Put everythin under phantom # We want to do it lazily too since we will be adding # data it can access later phantom = configFromMap({'phantom_tag': True, 'phantom': configToDict(configFromStream(open(fname + '.phantom'), lazy=True))}, lazy=True) else: phantom = configFromMap({}) ## # If the fname actually exists, open its meta data + files # if the fname does not exist but the phantom does, return the phantom # otherwise, throw an exception about missing the tagfile if os.path.exists(fname): if os.path.exists(fname + '.metadata'): metadata = configFromMap({'metadata': json.loads(open(fname + '.metadata').read())}, phantom, lazy=True) else: metadata = configFromMap({}, phantom) return configFromMap({'files': [f.strip() for f in open(fname) if f.strip()]}, metadata, lazy=True) elif not os.path.exists(fname) and os.path.exists(fname + '.phantom'): if os.path.exists(fname + '.metadata'): metadata = configFromMap({'metadata': json.loads(open(fname + '.metadata').read())}, phantom, lazy=True) return metadata else : return phantom else: raise MissingTagFileError(fname)
def instantiateCredential(conf, cred): """ Takes a credential and instanitates it. It returns a Record that has all of the information users of that instantiated credential will need """ if not conf("config_loaded", default=False): conf = config.configFromMap( {"config_loaded": True}, base=config.configFromStream(open(conf("general.conf_file", default=DEFAULT_CONFIG_FILE)), base=conf), ) certFile = os.path.join(conf("general.secure_tmp"), cred.name + "_cert.pem") keyFile = os.path.join(conf("general.secure_tmp"), cred.name + "_key.pem") if not os.path.exists(certFile) or open(certFile).read() != cred.cert: open(certFile, "w").write(cred.cert) if not os.path.exists(keyFile) or open(keyFile).read() != cred.pkey: open(keyFile, "w").write(cred.pkey) newCred = functional.Record(cert=certFile, pkey=keyFile, ec2URL=None, env={}) if "ec2_url" in cred.metadata: return (conf, newCred.update(env=functional.updateDict(newCred.env, dict(EC2_URL=cred.metadata["ec2_url"])))) else: return (conf, newCred)
def _loadTag(): tagPath = _createTagPath(self.conf, tagName) if not os.path.exists(tagPath) and not os.path.exists(tagPath + ".phantom"): return defer.fail(TagNotFoundError(tagName)) if os.path.exists(tagPath + ".phantom"): phantom = config.configFromStream(open(tagPath + ".phantom"), lazy=True) else: phantom = None if os.path.exists(tagPath + ".metadata"): metadata = json.loads(open(tagPath + ".metadata").read()) else: metadata = {} if os.path.exists(tagPath): files = [f.rstrip("\n") for f in open(tagPath).readlines() if f.strip()] else: files = [] return Tag( tagName=tagName, files=files, metadata=metadata, phantom=phantom, taskName=metadata.get("task_name") )
def _delayAutoshutdown(state, batchState): _log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file') if (('cluster_task' not in batchState or batchState.get('state', None) != RUNNING_STATE) and batchState.get('state', None) != COMPLETED_STATE): # Not ready yet _log(batchState, 'AUTOSHUTDOWN: No cluster or not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) elif ('cluster_task' in batchState and batchState.get('state', None) == RUNNING_STATE and batchState.get('pipeline_state', None) != SHUTDOWN_STATE): # Ready to see if resizing _log(batchState, 'AUTOSHUTDOWN: Making sure cluster is up') yield _blockOnTask(batchState['cluster_task']) try: cluster = yield loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') # We need the SSH options from the machine.conf, ugh I hate these OOB dependencies conf = config.configFromStream(open('/tmp/machine.conf')) _log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown') yield ssh.runProcessSSH(cluster['master']['public_dns'], 'touch /var/vappio/runtime/delayautoshutdown', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) except: pass _log(batchState, 'AUTOSHUTDOWN: Setting up next call') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) else: _log(batchState, 'AUTOSHUTDOWN: Giving up on this')
def _setQueue(taskName, batchState): yield _blockOnTask(taskName) cluster = yield loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield defer_utils.tryUntil(10, lambda : _getOutput(batchState, ['/opt/clovr_pipelines/workflow/project_saved_templates/clovr_lgt_wrapper/set_queue.sh', cluster['master']['public_dns']], log=True), onFailure=defer_utils.sleep(2)) conf = config.configFromStream(open('/tmp/machine.conf')) # Remove autoshutdown, we want no part of that yield ssh.runProcessSSH(cluster['master']['public_dns'], 'rm -v /var/vappio/runtime/noautoshutdown', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True)
def instantiateCredential(conf, cred): if not conf('config_loaded', default=False): conf = config.configFromConfig(conf, base=config.configFromStream(open( conf('conf_file')), base=conf)) certFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert.pem') keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem') mainDeferred = defer.succeed(None) if not os.path.exists(certFile) and not os.path.exists(keyFile): tmpCertFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert-tmp.pem') tmpKeyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key-tmp.pem') if 'ec2_url' not in cred.metadata: return defer.fail(Exception('You must have an ec2_url')) parsedUrl = urlparse.urlparse(cred.metadata['ec2_url']) if ':' not in parsedUrl.netloc: return defer.fail(Exception('Your URL must contain a port')) host, port = parsedUrl.netloc.split(':') fout = open(tmpCertFile, 'w') fout.write(cred.cert) fout.close() fout = open(tmpKeyFile, 'w') fout.write(cred.pkey) fout.close() d = commands.runProcess([ 'nimbusCerts2EC2.py', '--in-cert=' + tmpCertFile, '--out-cert=' + certFile, '--in-key=' + tmpKeyFile, '--out-key=' + keyFile, '--java-cert-dir=/tmp', '--java-cert-host=' + host, '--java-cert-port=' + port ], stdoutf=None, stderrf=None, log=True) def _chmod(_exitCode): return commands.runProcess(['chmod', '+r', keyFile], stdoutf=None, stderrf=None) d.addCallback(_chmod) def _unlink(v): os.unlink(tmpCertFile) os.unlink(tmpKeyFile) return v d.addCallback(_unlink) d.addErrback(_unlink) mainDeferred.addCallback(lambda _: d) ec2Home = cred.metadata.get('ec2_api_tools', '/opt/ec2-api-tools-1.3-57419') newCred = func.Record( name=cred.name, conf=conf, cert=certFile, pkey=keyFile, ec2Path=os.path.join(ec2Home, 'bin'), env=dict(EC2_JVM_ARGS='-Djavax.net.ssl.trustStore=/tmp/jssecacerts', EC2_HOME=ec2Home, EC2_URL=cred.metadata['ec2_url'])) if os.path.exists(conf('cluster.cluster_private_key') + '.pub'): pubKey = open(conf('cluster.cluster_private_key') + '.pub').read().rstrip() def _addKeypair(): keyPairDefer = ec2.addKeypair(newCred, conf('cluster.key') + '||' + pubKey) def _printError(f): log.msg('Adding keypair failed, retrying') log.err(f) return f keyPairDefer.addErrback(_printError) return keyPairDefer mainDeferred.addCallback(lambda _: defer_utils.tryUntil( 10, _addKeypair, onFailure=defer_utils.sleep(30))) mainDeferred.addCallback(lambda _: newCred) return mainDeferred
def _run(state, batchState): if 'pipeline_name' in batchState: pipelines = yield pipelines_client.pipelineList( 'localhost', 'local', 'guest', batchState['pipeline_name'], detail=True) else: pipelines = [] if not pipelines: _log(batchState, 'First time running, creating pipeline state information') batchState['pipeline_config'] = yield _applyActions( state.innerPipelineConfig(), batchState['actions']) batchState['pipeline_state'] = STARTCLUSTER_STATE # We need to create a fake, local, pipeline for metrics to work batchState['pipeline_name'] = pipeline_misc.checksumInput( batchState['pipeline_config']) batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState[ 'pipeline_name'] batchState['pipeline_config'][ 'pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name'] _log(batchState, 'Pipeline named ' + batchState['pipeline_name']) pipeline = yield pipelines_client.createPipeline( host='localhost', clusterName='local', userName='******', pipelineName=batchState['pipeline_name'], protocol='clovr_wrapper', queue='pipeline.q', config=batchState['pipeline_config'], parentPipeline=state.parentPipeline()) batchState['lgt_wrapper_task_name'] = pipeline['task_name'] _log( batchState, 'Setting number of tasks to 9 (number in a standard lgt_wrapper)') yield _updateTask(batchState, lambda t: t.update(completedTasks=0, numTasks=9)) state.updateBatchState() batchState['state'] = RUNNING_STATE _log(batchState, 'Pipeline started in %s state' % batchState['pipeline_state']) yield _updateTask(batchState, lambda t: t.setState(tasks.task.TASK_RUNNING)) pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs', global_state.make_ref() + '.conf') _log(batchState, 'Creating ergatis configuration') _writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile) if batchState['pipeline_state'] == STARTCLUSTER_STATE: _log(batchState, 'Pipeline is in STARTCLUSTER state') # First see if the cluster exists but is unresponsive try: cluster = yield loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, terminating') terminateTask = yield clusters_client.terminateCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _blockOnTask(terminateTask) except: pass batchState['cluster_task'] = yield startCluster( batchState, 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', int(batchState['pipeline_config']['cluster.EXEC_NODES']), 0, batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'], { 'cluster.MASTER_INSTANCE_TYPE': batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'], 'cluster.MASTER_BID_PRICE': batchState['pipeline_config']['cluster.MASTER_BID_PRICE'], 'cluster.EXEC_INSTANCE_TYPE': batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'], 'cluster.EXEC_BID_PRICE': batchState['pipeline_config']['cluster.EXEC_BID_PRICE'] }) yield _updateTask(batchState, lambda t: t.update(completedTasks=0, numTasks=9)) yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed startcluster').progress()) batchState['pipeline_state'] = REMOTE_LOCAL_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REMOTE_LOCAL_TRANSFER_STATE: _log(batchState, 'Pipeline is in REMOTE_LOCAL_TRANSFER') _log(batchState, 'Making sure cluster is exists in some form') cluster = yield clusters_client.loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, erroring and restarting') raise Exception('Cluster is not responsive') yield state.prerunQueue.addWithDeferred(_remoteLocalTransfer, batchState) yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed remote_local_transfer'). progress()) batchState['pipeline_state'] = DECRYPT_STATE state.updateBatchState() if batchState['pipeline_state'] == DECRYPT_STATE: _log(batchState, 'Pipeline is in DECRYPT') cluster = yield loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') tag = yield tags_client.loadTag( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', _decryptTagName(batchState)) conf = config.configFromStream(open('/tmp/machine.conf')) yield ssh.runProcessSSH(cluster['master']['public_dns'], 'mkdir -p /mnt/lgt_decrypt', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) yield rsync.rsyncTo( cluster['master']['public_dns'], batchState['pipeline_config']['params.DECRYPT_SCRIPT'], '/mnt/', options=conf('rsync.options'), user=conf('rsync.user'), log=True) for f in tag['files']: decryptCmd = ' '.join([ os.path.join( '/mnt', os.path.basename(batchState['pipeline_config'] ['params.DECRYPT_SCRIPT'])), f, '-out-dir', '/mnt/lgt_decrypt', '-remove-encrypted', '-password', batchState['pipeline_config']['params.DECRYPT_PASSWORD'] ]) yield ssh.getOutput(cluster['master']['public_dns'], decryptCmd, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), expected=[0, 253], log=True) tag = yield tags_client.tagData( host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', action='overwrite', tagName=_decryptTagName(batchState), files=['/mnt/lgt_decrypt'], metadata={}, recursive=True, expand=False, compressDir=None) _log( batchState, 'Waiting for tagging of %s to complete - %s' % (_decryptTagName(batchState), tag['task_name'])) yield _blockOnTask( tag['task_name'], cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME']) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed decrypt').progress()) batchState['pipeline_state'] = REFERENCE_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REFERENCE_TRANSFER_STATE: _log(batchState, 'Pipeline is in REFERENCE_TRANSFER state') transfers = [] tags = (batchState['pipeline_config']['input.REF_TAG1'].split(',') + batchState['pipeline_config']['input.REF_TAG2'].split(',')) for tag in tags: tag = tag.strip() output = yield _getOutput(batchState, [ 'vp-transfer-dataset', '-t', '--tag-name=' + tag, '--dst-cluster=' + batchState['pipeline_config']['cluster.CLUSTER_NAME'] ], log=True) transfers.append(output['stdout'].strip()) for task in transfers: yield _blockOnTask(task) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed reference_transfer').progress()) batchState['pipeline_state'] = RUN_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUN_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUN_PIPELINE state') batchState['pipeline_config']['input.INPUT_TAG'] = _decryptTagName( batchState) pipeline = yield pipelines_client.runPipeline( host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', parentPipeline=batchState['pipeline_name'], bareRun=True, queue=state.innerPipelineQueue(), config=batchState['pipeline_config'], overwrite=True) batchState['pipeline_task'] = pipeline['task_name'] yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed run pipeline').progress()) batchState['pipeline_state'] = RUNNING_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUNNING_PIPELINE state') _monitorPipeline(batchState) yield _waitForPipeline(batchState) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed running pipeline').progress()) batchState['pipeline_state'] = HARVEST_STATE state.updateBatchState() if batchState['pipeline_state'] == HARVEST_STATE: _log(batchState, 'Pipeline is in HARVEST state') # Using prerunqueue because we want everything here serialized yield state.prerunQueue.addWithDeferred(_harvestTransfer, batchState) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed harvest').progress()) batchState['pipeline_state'] = SHUTDOWN_STATE state.updateBatchState() if batchState['pipeline_state'] == SHUTDOWN_STATE: _log(batchState, 'Pipeline is in SHUTDOWN state') if 'add_instances_task' in batchState: try: yield _blockOnTask(batchState['add_instances_task'], cluster=batchState['pipeline_config'] ['cluster.CLUSTER_NAME']) except Exception, err: logging.errorPrint(str(err)) log.err(err) yield clusters_client.terminateCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed shutdown').progress()) batchState['pipeline_state'] = COMPLETED_STATE batchState['state'] = COMPLETED_STATE state.updateBatchState()
def instantiateCredential(conf, cred): if not conf('config_loaded', default=False): conf = config.configFromMap({'config_loaded': True}, base=config.configFromStream(open(conf('general.conf_file', default=DEFAULT_CONFIG_FILE)), base=conf)) return (conf, None)
from twisted.application import service from twisted.python import log from igs.utils import config from vappio_tx.mq import client from vappio_tx.internal_client import credentials conf = config.configFromStream(open('/mnt/vappio-conf/vappio_apps.conf')) application = service.Application('test') s1 = client.makeService(conf) s1.setServiceParent(application) cc = credentials.CredentialClient('diag', s1.mqFactory, conf) d = cc.listInstances() def _terminate(instances): print 'Num instances:', len(instances) instances = instances[:3] print 'Shutting down: ', instances return cc.terminateInstances(instances) def _print(foo): print 'Foo:', foo
def buildConfigN(options, args=None, usage=None, baseConf=None, putInGeneral=True): """ This builds a config from options. Options is a list of tuples that looks like: (name, short, long, help, func, [bool]) Where name - Name of the option, this is what it will become in the config file short - Short option - needs to start with - long - Long option - needs to start with -- help - Help to be given to a user in --help output func - Function to be applied to the value bool - This is not required, set to True if the option is simply a boolean, all other datatypes can be verified via 'func' This will implicitly check if a 'conf' option exists, and if so load te conf file as a base for these config options. All options are put into the 'general' section. This returns a tuple (conf, args) where args is whatever is left over from parsing This also implicitly loads the current environment into the env section If, when evaluated, 'func' returns a function, it is called with the baseConf. This is to allow more complex replacements to happen. """ def _iterBool(v): """ Adds the non erquired bool field with a default of STRING if it is not present """ for l in v: if len(l) == 6: yield l else: yield tuple(list(l) + [STRING]) parser = optparse.OptionParser(usage=usage) ## # keep track of the function to apply to conf confFunc = None ## # The order of the options below # var name, short option, long option, help message, function to apply, binary option for n, s, l, h, f, b in _iterBool(options): ## # We could have a function we want to apply to the conf variable. We want to store it # so when we use it in the next block we don't have to loop over options looking for it again # This is a minor optimization and probably not even necessary... if n == 'conf': confFunc = f if b == BINARY: parser.add_option(s, l, dest=n, help=h, action='store_true') elif b == LIST: parser.add_option(s, l, dest=n, help=h, action='append') elif b == COUNT: parser.add_option(s, l, dest=n, help=h, action='count') elif b == STRING: parser.add_option(s, l, dest=n, help=h) else: raise Exception('Unknown option type: ' + repr(b)) ops, args = parser.parse_args(args=args) if baseConf is None: baseConf = configFromEnv() if hasattr(ops, 'conf'): baseConf = configFromStream( open(replaceStr(confFunc(ops.conf), baseConf)), baseConf) vals = {} ## # The order of the options below # var name, short option, long option, help message, function to apply, binary option for o in _iterBool(options): n, _s, l, _h, f, _b = o try: vals[n] = applyOption(getattr(ops, n), o, baseConf) except Exception, err: raise CLIError(l, err)
from twisted.application import service from twisted.python import log from igs.utils import config from vappio_tx.mq import client from vappio_tx.internal_client import credentials conf = config.configFromStream(open('/mnt/vappio-conf/vappio_apps.conf')) application = service.Application('test') s1 = client.makeService(conf) s1.setServiceParent(application) cc = credentials.CredentialClient('diag', s1.mqFactory, conf) d = cc.listInstances() def _terminate(instances): print 'Num instances:', len(instances) instances = instances[:3] print 'Shutting down: ', instances return cc.terminateInstances(instances) def _print(foo): print 'Foo:', foo d.addCallback(_terminate) d.addCallback(_print)
def instantiateCredential(conf, cred): if not conf('config_loaded', default=False): conf = config.configFromConfig(conf, base=config.configFromStream(open(conf('conf_file')), base=conf)) certFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert.pem') keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem') mainDeferred = defer.succeed(None) if not os.path.exists(certFile) and not os.path.exists(keyFile): tmpCertFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert-tmp.pem') tmpKeyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key-tmp.pem') if 'ec2_url' not in cred.metadata: return defer.fail(Exception('You must have an ec2_url')) parsedUrl = urlparse.urlparse(cred.metadata['ec2_url']) if ':' not in parsedUrl.netloc: return defer.fail(Exception('Your URL must contain a port')) host, port = parsedUrl.netloc.split(':') fout = open(tmpCertFile, 'w') fout.write(cred.cert) fout.close() fout = open(tmpKeyFile, 'w') fout.write(cred.pkey) fout.close() d = commands.runProcess(['nimbusCerts2EC2.py', '--in-cert=' + tmpCertFile, '--out-cert=' + certFile, '--in-key=' + tmpKeyFile, '--out-key=' + keyFile, '--java-cert-dir=/tmp', '--java-cert-host=' + host, '--java-cert-port=' + port], stdoutf=None, stderrf=None, log=True) def _chmod(_exitCode): return commands.runProcess(['chmod', '+r', keyFile], stdoutf=None, stderrf=None) d.addCallback(_chmod) def _unlink(v): os.unlink(tmpCertFile) os.unlink(tmpKeyFile) return v d.addCallback(_unlink) d.addErrback(_unlink) mainDeferred.addCallback(lambda _ : d) ec2Home = cred.metadata.get('ec2_api_tools', '/opt/ec2-api-tools-1.3-57419') newCred = func.Record(name=cred.name, conf=conf, cert=certFile, pkey=keyFile, ec2Path=os.path.join(ec2Home, 'bin'), env=dict(EC2_JVM_ARGS='-Djavax.net.ssl.trustStore=/tmp/jssecacerts', EC2_HOME=ec2Home, EC2_URL=cred.metadata['ec2_url'])) if os.path.exists(conf('cluster.cluster_private_key') + '.pub'): pubKey = open(conf('cluster.cluster_private_key') + '.pub').read().rstrip() def _addKeypair(): keyPairDefer = ec2.addKeypair(newCred, conf('cluster.key') + '||' + pubKey) def _printError(f): log.msg('Adding keypair failed, retrying') log.err(f) return f keyPairDefer.addErrback(_printError) return keyPairDefer mainDeferred.addCallback(lambda _ : defer_utils.tryUntil(10, _addKeypair, onFailure=defer_utils.sleep(30))) mainDeferred.addCallback(lambda _ : newCred) return mainDeferred
def buildConfigN(options, args=None, usage=None, baseConf=None, putInGeneral=True): """ This builds a config from options. Options is a list of tuples that looks like: (name, short, long, help, func, [bool]) Where name - Name of the option, this is what it will become in the config file short - Short option - needs to start with - long - Long option - needs to start with -- help - Help to be given to a user in --help output func - Function to be applied to the value bool - This is not required, set to True if the option is simply a boolean, all other datatypes can be verified via 'func' This will implicitly check if a 'conf' option exists, and if so load te conf file as a base for these config options. All options are put into the 'general' section. This returns a tuple (conf, args) where args is whatever is left over from parsing This also implicitly loads the current environment into the env section If, when evaluated, 'func' returns a function, it is called with the baseConf. This is to allow more complex replacements to happen. """ def _iterBool(v): """ Adds the non erquired bool field with a default of STRING if it is not present """ for l in v: if len(l) == 6: yield l else: yield tuple(list(l) + [STRING]) parser = optparse.OptionParser(usage=usage) ## # keep track of the function to apply to conf confFunc = None ## # The order of the options below # var name, short option, long option, help message, function to apply, binary option for n, s, l, h, f, b in _iterBool(options): ## # We could have a function we want to apply to the conf variable. We want to store it # so when we use it in the next block we don't have to loop over options looking for it again # This is a minor optimization and probably not even necessary... if n == 'conf': confFunc = f if b == BINARY: parser.add_option(s, l, dest=n, help=h, action='store_true') elif b == LIST: parser.add_option(s, l, dest=n, help=h, action='append') elif b == COUNT: parser.add_option(s, l, dest=n, help=h, action='count') elif b == STRING: parser.add_option(s, l, dest=n, help=h) else: raise Exception('Unknown option type: ' + repr(b)) ops, args = parser.parse_args(args=args) if baseConf is None: baseConf = configFromEnv() if hasattr(ops, 'conf'): baseConf = configFromStream(open(replaceStr(confFunc(ops.conf), baseConf)), baseConf) vals = {} ## # The order of the options below # var name, short option, long option, help message, function to apply, binary option for o in _iterBool(options): n, _s, l, _h, f, _b = o try: vals[n] = applyOption(getattr(ops, n), o, baseConf) except Exception, err: raise CLIError(l, err)
## # These define useful policies for ensuring a machine is configed properly. This should probably be off lifted into # a program/library designed for this, but what I need right now is simple import os from igs.utils.logging import errorPrintS from igs.utils.commands import runSingleProgram, ProgramRunError from igs.utils.config import configFromMap, configFromStream, configFromEnv, replaceStr ## # These are default config options, these will be moved to a config file eventually conf = configFromStream(open('/tmp/machine.conf'), configFromMap({ 'stow': {'package_dir': '/usr/local/stow', 'base_dir': '/usr/local'}, 'opt': {'package_dir': '/opt/opt-packages', 'base_dir': '/opt'}, 'config': {'filename': '/tmp/machine.conf'}, }, configFromEnv())) ## # Exceptions class PolicyError(Exception): pass ## # A little helper function def runSystemEx(cmd):
def configFromStream(stream): return fixVariables(config.configFromStream(stream, config.configFromEnv()))
def loadLocalCluster(mq, state): """ If local cluster is not present, load it """ def _credential(): if os.path.exists('/tmp/cred-info'): cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split('\t') return {'name': 'local', 'desc': 'Local credential', 'ctype': ctype, 'cert': open(cert).read(), 'pkey': open(pkey).read(), 'metadata': metadata and dict([v.split('=', 1) for v in metadata.split(',')]) or {}, 'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True)} else: return {'name': 'local', 'desc': 'Local credential', 'ctype': 'local', 'cert': None, 'pkey': None, 'metadata': {}, 'conf': config.configFromMap({})} try: cluster = yield state.persistManager.loadCluster('local', None) baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap({'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'}, base=baseConf) if (cluster.credName == 'local' and conf('MASTER_IP') not in [cluster.master['public_dns'], cluster.master['private_dns']]): master = dict(instance_id='local', ami_id=None, public_dns=conf('MASTER_IP'), private_dns=conf('MASTER_IP'), state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master).update(config=conf) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster) except persist.ClusterNotFoundError: credential = _credential() credTaskName = yield cred_client.saveCredential(credential['name'], credential['desc'], credential['ctype'], credential['cert'], credential['pkey'], credential['metadata'], credential['conf']) ## Wait for credential to be added. ## TODO: Should handle failure here yield tasks_tx.blockOnTask('localhost', 'local', credTaskName) credClient = cred_client.CredentialClient('local', mq, state.conf) ## If it isn't a local ctype then we need to wait for ## the credential to come alive if credential['ctype'] != 'local': instances = yield credClient.listInstances() else: instances = [] baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap({'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'}, base=baseConf) cluster = persist.Cluster('local', None, 'local', conf) startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1) yield tasks_tx.updateTask(startTaskName, lambda t : t.setState(tasks_tx.task.TASK_COMPLETED).progress()) cluster = cluster.update(startTask=startTaskName) masterIp = cluster.config('MASTER_IP') masterIdx = func.find(lambda i : masterIp in [i['public_dns'], i['private_dns']], instances) if masterIdx is not None: master = instances[masterIdx] else: master = dict(instance_id='local', ami_id=None, public_dns=masterIp, private_dns=masterIp, state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master) cluster = cluster.setState(cluster.RUNNING) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster)
def _run(state, batchState): if 'pipeline_name' in batchState: pipelines = yield pipelines_client.pipelineList('localhost', 'local', 'guest', batchState['pipeline_name'], detail=True) else: pipelines = [] if not pipelines: _log(batchState, 'First time running, creating pipeline state information') batchState['pipeline_config'] = yield _applyActions(state.innerPipelineConfig(), batchState['actions']) batchState['pipeline_state'] = STARTCLUSTER_STATE # We need to create a fake, local, pipeline for metrics to work batchState['pipeline_name'] = pipeline_misc.checksumInput(batchState['pipeline_config']) batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState['pipeline_name'] batchState['pipeline_config']['pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name'] _log(batchState, 'Pipeline named ' + batchState['pipeline_name']) pipeline = yield pipelines_client.createPipeline(host='localhost', clusterName='local', userName='******', pipelineName=batchState['pipeline_name'], protocol='clovr_wrapper', queue='pipeline.q', config=batchState['pipeline_config'], parentPipeline=state.parentPipeline()) batchState['lgt_wrapper_task_name'] = pipeline['task_name'] _log(batchState, 'Setting number of tasks to 9 (number in a standard lgt_wrapper)') yield _updateTask(batchState, lambda t : t.update(completedTasks=0, numTasks=9)) state.updateBatchState() batchState['state'] = RUNNING_STATE _log(batchState, 'Pipeline started in %s state' % batchState['pipeline_state']) yield _updateTask(batchState, lambda t : t.setState(tasks.task.TASK_RUNNING)) pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs', global_state.make_ref() + '.conf') _log(batchState, 'Creating ergatis configuration') _writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile) if batchState['pipeline_state'] == STARTCLUSTER_STATE: _log(batchState, 'Pipeline is in STARTCLUSTER state') # First see if the cluster exists but is unresponsive try: cluster = yield loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, terminating') terminateTask = yield clusters_client.terminateCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _blockOnTask(terminateTask) except: pass batchState['cluster_task'] = yield startCluster( batchState, 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', int(batchState['pipeline_config']['cluster.EXEC_NODES']), 0, batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'], {'cluster.MASTER_INSTANCE_TYPE': batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'], 'cluster.MASTER_BID_PRICE': batchState['pipeline_config']['cluster.MASTER_BID_PRICE'], 'cluster.EXEC_INSTANCE_TYPE': batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'], 'cluster.EXEC_BID_PRICE': batchState['pipeline_config']['cluster.EXEC_BID_PRICE']}) yield _updateTask(batchState, lambda t : t.update(completedTasks=0, numTasks=9)) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed startcluster' ).progress()) batchState['pipeline_state'] = REMOTE_LOCAL_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REMOTE_LOCAL_TRANSFER_STATE: _log(batchState, 'Pipeline is in REMOTE_LOCAL_TRANSFER') _log(batchState, 'Making sure cluster is exists in some form') cluster = yield clusters_client.loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, erroring and restarting') raise Exception('Cluster is not responsive') yield state.prerunQueue.addWithDeferred(_remoteLocalTransfer, batchState) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed remote_local_transfer' ).progress()) batchState['pipeline_state'] = DECRYPT_STATE state.updateBatchState() if batchState['pipeline_state'] == DECRYPT_STATE: _log(batchState, 'Pipeline is in DECRYPT') cluster = yield loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') tag = yield tags_client.loadTag('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', _decryptTagName(batchState)) conf = config.configFromStream(open('/tmp/machine.conf')) yield ssh.runProcessSSH(cluster['master']['public_dns'], 'mkdir -p /mnt/lgt_decrypt', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) yield rsync.rsyncTo(cluster['master']['public_dns'], batchState['pipeline_config']['params.DECRYPT_SCRIPT'], '/mnt/', options=conf('rsync.options'), user=conf('rsync.user'), log=True) for f in tag['files']: decryptCmd = ' '.join([os.path.join('/mnt', os.path.basename(batchState['pipeline_config']['params.DECRYPT_SCRIPT'])), f, '-out-dir', '/mnt/lgt_decrypt', '-remove-encrypted', '-password', batchState['pipeline_config']['params.DECRYPT_PASSWORD']]) yield ssh.getOutput(cluster['master']['public_dns'], decryptCmd, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), expected=[0, 253], log=True) tag = yield tags_client.tagData(host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', action='overwrite', tagName=_decryptTagName(batchState), files=['/mnt/lgt_decrypt'], metadata={}, recursive=True, expand=False, compressDir=None) _log(batchState, 'Waiting for tagging of %s to complete - %s' % (_decryptTagName(batchState), tag['task_name'])) yield _blockOnTask(tag['task_name'], cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME']) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed decrypt' ).progress()) batchState['pipeline_state'] = REFERENCE_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REFERENCE_TRANSFER_STATE: _log(batchState, 'Pipeline is in REFERENCE_TRANSFER state') transfers = [] tags = (batchState['pipeline_config']['input.REF_TAG1'].split(',') + batchState['pipeline_config']['input.REF_TAG2'].split(',')) for tag in tags: tag = tag.strip() output = yield _getOutput(batchState, ['vp-transfer-dataset', '-t', '--tag-name=' + tag, '--dst-cluster=' + batchState['pipeline_config']['cluster.CLUSTER_NAME']], log=True) transfers.append(output['stdout'].strip()) for task in transfers: yield _blockOnTask(task) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed reference_transfer' ).progress()) batchState['pipeline_state'] = RUN_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUN_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUN_PIPELINE state') batchState['pipeline_config']['input.INPUT_TAG'] = _decryptTagName(batchState) pipeline = yield pipelines_client.runPipeline(host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', parentPipeline=batchState['pipeline_name'], bareRun=True, queue=state.innerPipelineQueue(), config=batchState['pipeline_config'], overwrite=True) batchState['pipeline_task'] = pipeline['task_name'] yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed run pipeline' ).progress()) batchState['pipeline_state'] = RUNNING_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUNNING_PIPELINE state') _monitorPipeline(batchState) yield _waitForPipeline(batchState) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed running pipeline' ).progress()) batchState['pipeline_state'] = HARVEST_STATE state.updateBatchState() if batchState['pipeline_state'] == HARVEST_STATE: _log(batchState, 'Pipeline is in HARVEST state') # Using prerunqueue because we want everything here serialized yield state.prerunQueue.addWithDeferred(_harvestTransfer, batchState) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed harvest' ).progress()) batchState['pipeline_state'] = SHUTDOWN_STATE state.updateBatchState() if batchState['pipeline_state'] == SHUTDOWN_STATE: _log(batchState, 'Pipeline is in SHUTDOWN state') if 'add_instances_task' in batchState: try: yield _blockOnTask(batchState['add_instances_task'], cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME']) except Exception, err: logging.errorPrint(str(err)) log.err(err) yield clusters_client.terminateCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed shutdown' ).progress()) batchState['pipeline_state'] = COMPLETED_STATE batchState['state'] = COMPLETED_STATE state.updateBatchState()
import os from igs.utils.logging import errorPrintS from igs.utils.commands import runSingleProgram, ProgramRunError from igs.utils.config import configFromMap, configFromStream, configFromEnv, replaceStr ## # These are default config options, these will be moved to a config file eventually conf = configFromStream( open('/tmp/machine.conf'), configFromMap( { 'stow': { 'package_dir': '/usr/local/stow', 'base_dir': '/usr/local' }, 'opt': { 'package_dir': '/opt/opt-packages', 'base_dir': '/opt' }, 'config': { 'filename': '/tmp/machine.conf' }, }, configFromEnv())) ## # Exceptions class PolicyError(Exception): pass
def loadLocalCluster(mq, state): """ If local cluster is not present, load it """ def _credential(): if os.path.exists('/tmp/cred-info'): cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split( '\t') return { 'name': 'local', 'desc': 'Local credential', 'ctype': ctype, 'cert': open(cert).read(), 'pkey': open(pkey).read(), 'metadata': metadata and dict([v.split('=', 1) for v in metadata.split(',')]) or {}, 'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True) } else: return { 'name': 'local', 'desc': 'Local credential', 'ctype': 'local', 'cert': None, 'pkey': None, 'metadata': {}, 'conf': config.configFromMap({}) } try: cluster = yield state.persistManager.loadCluster('local', None) baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap( { 'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub' }, base=baseConf) if (cluster.credName == 'local' and conf('MASTER_IP') not in [ cluster.master['public_dns'], cluster.master['private_dns'] ]): master = dict(instance_id='local', ami_id=None, public_dns=conf('MASTER_IP'), private_dns=conf('MASTER_IP'), state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master).update(config=conf) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster) except persist.ClusterNotFoundError: credential = _credential() credTaskName = yield cred_client.saveCredential( credential['name'], credential['desc'], credential['ctype'], credential['cert'], credential['pkey'], credential['metadata'], credential['conf']) ## Wait for credential to be added. ## TODO: Should handle failure here yield tasks_tx.blockOnTask('localhost', 'local', credTaskName) credClient = cred_client.CredentialClient('local', mq, state.conf) ## If it isn't a local ctype then we need to wait for ## the credential to come alive if credential['ctype'] != 'local': instances = yield credClient.listInstances() else: instances = [] baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap( { 'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub' }, base=baseConf) cluster = persist.Cluster('local', None, 'local', conf) startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1) yield tasks_tx.updateTask( startTaskName, lambda t: t.setState(tasks_tx.task.TASK_COMPLETED).progress()) cluster = cluster.update(startTask=startTaskName) masterIp = cluster.config('MASTER_IP') masterIdx = func.find( lambda i: masterIp in [i['public_dns'], i['private_dns']], instances) if masterIdx is not None: master = instances[masterIdx] else: master = dict(instance_id='local', ami_id=None, public_dns=masterIp, private_dns=masterIp, state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master) cluster = cluster.setState(cluster.RUNNING) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster)