def run(options):
logging.logPrint('Starting')
batchConfig = config.configFromStream(open(options.configFile), lazy=True)
machineConf = config.configFromStream(open('/tmp/machine.conf'))
state = State(options.workflowConfig,
options.batchStatesFile,
_validateWrapper(batchConfig('batch_pipeline.pipeline.PIPELINE_TEMPLATE'),
pipeline_misc.determineWrapper(machineConf,
batchConfig('batch_pipeline.pipeline.PIPELINE_TEMPLATE'))),
_interpretBatchFile(options.batchFile),
_extractInnerPipelineConfig(batchConfig),
batchConfig('pipeline.PIPELINE_WRAPPER_NAME'),
int(batchConfig('batch.options.CONCURRENT_PRERUN')),
int(batchConfig('batch.options.CONCURRENT_PIPELINES')),
int(batchConfig('batch.options.CONCURRENT_POSTRUN')))
logging.logPrint('Queuing any incomplete work')
queueCount = _queueIncompleteWork(state)
logging.logPrint('Queued: %d' % queueCount)
if state.pipelinesQueue.hasWork():
yield defer_work_queue.waitForCompletion(state.pipelinesQueue)
for batchState in state.batchStates.values():
if 'state' not in batchState or batchState['state'] == 'failed':
raise JobFailed()
def confIfPipelineConfigSet(conf, options):
"""
Takes a conf, checks to see if a pipeline conf file is specified,
if so it loads it up and applies it OVER any options specified on
the command line. This may seem counter intuitive but it makes
other things easier, for example a pipeline redefining anything
in the machines.conf since that is also in this conf. It then
applies the functions in the OPTIONS variable in the values in
the config file
"""
if conf('CONFIG_FILE', default=None) is not None:
fconf = config.configFromStream(open(conf('CONFIG_FILE')))
keys = fconf.keys()
m = {}
for o in options:
##
# Get the name of the option, it's the first element of the tuple
name = o[0]
f = o[4]
if name in keys:
m[name] = applyIfCallable(f(fconf(name)), conf)
##
# lazy=True is for saftey incase there is a value in the CONFIG_FILE that we use that
# really depends on a value in the map we just created
return config.configFromMap(
m,
config.configFromStream(open(conf('CONFIG_FILE')),
conf,
lazy=False))
else:
return conf
def confIfPipelineConfigSet(conf, options):
"""
Takes a conf, checks to see if a pipeline conf file is specified,
if so it loads it up and applies it OVER any options specified on
the command line. This may seem counter intuitive but it makes
other things easier, for example a pipeline redefining anything
in the machines.conf since that is also in this conf. It then
applies the functions in the OPTIONS variable in the values in
the config file
"""
if conf("CONFIG_FILE", default=None) is not None:
fconf = config.configFromStream(open(conf("CONFIG_FILE")))
keys = fconf.keys()
m = {}
for o in options:
##
# Get the name of the option, it's the first element of the tuple
name = o[0]
f = o[4]
if name in keys:
m[name] = applyIfCallable(f(fconf(name)), conf)
##
# lazy=True is for saftey incase there is a value in the CONFIG_FILE that we use that
# really depends on a value in the map we just created
return config.configFromMap(m, config.configFromStream(open(conf("CONFIG_FILE")), conf, lazy=False))
else:
return conf
def run(options):
logging.logPrint('Starting')
batchConfig = config.configFromStream(open(options.configFile), lazy=True)
machineConf = config.configFromStream(open('/tmp/machine.conf'))
state = State(
options.workflowConfig, options.batchStatesFile,
_validateWrapper(
batchConfig('batch_pipeline.pipeline.PIPELINE_TEMPLATE'),
pipeline_misc.determineWrapper(
machineConf,
batchConfig('batch_pipeline.pipeline.PIPELINE_TEMPLATE'))),
_interpretBatchFile(options.batchFile),
_extractInnerPipelineConfig(batchConfig),
batchConfig('pipeline.PIPELINE_WRAPPER_NAME'),
int(batchConfig('batch.options.CONCURRENT_PRERUN')),
int(batchConfig('batch.options.CONCURRENT_PIPELINES')),
int(batchConfig('batch.options.CONCURRENT_POSTRUN')))
logging.logPrint('Queuing any incomplete work')
queueCount = _queueIncompleteWork(state)
logging.logPrint('Queued: %d' % queueCount)
if state.pipelinesQueue.hasWork():
yield defer_work_queue.waitForCompletion(state.pipelinesQueue)
for batchState in state.batchStates.values():
if 'state' not in batchState or batchState['state'] == 'failed':
raise JobFailed()
def main(options, args):
if not options('general.pipeline_resume'):
conf = config.configFromStream(open(options('general.pipeline_config')), lazy=True)
ret = pipeline.validatePipelineConfig(options('general.host'),
options('general.cluster'),
options('general.bare_run'),
conf)
if ret['errors']:
for e in ret['errors']:
print '\t'.join(['ERROR', ','.join(e['keys']), e['message']])
# Exit with an error
sys.exit(1)
if not options('general.validate'):
p = pipeline.runPipeline(options('general.host'),
options('general.cluster'),
options('general.pipeline_parent'),
options('general.bare_run'),
conf,
options('general.pipeline_queue'),
options('general.overwrite'))
if options('general.print_task_name'):
print p['task_name']
else:
runTaskStatus(p['task_name'])
else:
p = pipeline.resumePipeline(options('general.host'),
options('general.cluster'),
options('general.pipeline_resume'))
if options('general.print_task_name'):
print p['task_name']
else:
runTaskStatus(p['task_name'])
def _delayAutoshutdown(state, batchState):
_log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file')
try:
clusters = yield clusters_client.listClusters('localhost',
{'cluster_name':
batchState['pipeline_config']['cluster.CLUSTER_NAME']},
'guest')
cluster = clusters[0]
if batchState.get('state', None) == COMPLETED_STATE:
_log(batchState, 'AUTOSHUTDOWN: Pipeline complete, done')
if batchState.get('state', None) != RUNNING_STATE:
_log(batchState, 'AUTOSHUTDOWN: Pipeline not running, calling later')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
elif cluster['state'] == 'running':
# We need the SSH options from the machine.conf, ugh I hate these OOB dependencies
conf = config.configFromStream(open('/tmp/machine.conf'))
_log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown')
yield ssh.runProcessSSH(cluster['master']['public_dns'],
'touch /var/vappio/runtime/delayautoshutdown',
stdoutf=None,
stderrf=None,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
log=True)
_log(batchState, 'AUTOSHUTDOWN: Setting up next call')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
else:
_log(batchState, 'AUTOSHUTDOWN: Cluster not running, calling later')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
except:
_log(batchState, 'AUTOSHUTDOWN: Cluster does not exist, calling later')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
def instantiateCredential(conf, cred):
"""
Takes a credential and instanitates it. It returns a Record that has all of the
information users of that instantiated credential will need
"""
if not conf('config_loaded', default=False):
conf = config.configFromConfig(
conf,
base=config.configFromStream(
open(conf('conf_file', default=DEFAULT_CONFIG_FILE)),
base=conf))
certFile = os.path.join(
conf('general.secure_tmp'), cred.name + '_cert.pem')
keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem')
if not os.path.exists(certFile) or open(certFile).read() != cred.cert:
open(certFile, 'w').write(cred.cert)
if not os.path.exists(keyFile) or open(keyFile).read() != cred.pkey:
open(keyFile, 'w').write(cred.pkey)
newCred = functional.Record(
name=cred.name,
conf=conf,
cert=certFile,
pkey=keyFile,
ec2URL=None,
env={})
if 'ec2_url' in cred.metadata:
newCred = newCred.update(
env=functional.updateDict(newCred.env,
dict(EC2_URL=cred.metadata['ec2_url'])))
yield _createGroups(newCred)
yield _createKeypair(newCred)
defer.returnValue(newCred)
def instantiateCredential(conf, cred):
"""
Takes a credential and instanitates it. It returns a Record that has all of the
information users of that instantiated credential will need
"""
if not conf('config_loaded', default=False):
conf = config.configFromMap({'config_loaded': True},
base=config.configFromStream(open(
conf('general.conf_file',
default=DEFAULT_CONFIG_FILE)),
base=conf))
certFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_cert.pem')
keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem')
if not os.path.exists(certFile) or open(certFile).read() != cred.cert:
open(certFile, 'w').write(cred.cert)
if not os.path.exists(keyFile) or open(keyFile).read() != cred.pkey:
open(keyFile, 'w').write(cred.pkey)
newCred = functional.Record(cert=certFile,
pkey=keyFile,
ec2URL=None,
env={})
if 'ec2_url' in cred.metadata:
return (conf,
newCred.update(env=functional.updateDict(
newCred.env, dict(EC2_URL=cred.metadata['ec2_url']))))
else:
return (conf, newCred)
def _setQueue(taskName, batchState):
yield _blockOnTask(taskName)
cluster = yield loadCluster(
'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
yield defer_utils.tryUntil(
10,
lambda: _getOutput(batchState, [
'/opt/clovr_pipelines/workflow/project_saved_templates/clovr_lgt_wrapper/set_queue.sh',
cluster['master']['public_dns']
],
log=True),
onFailure=defer_utils.sleep(2))
conf = config.configFromStream(open('/tmp/machine.conf'))
# Remove autoshutdown, we want no part of that
yield ssh.runProcessSSH(cluster['master']['public_dns'],
'rm -v /var/vappio/runtime/noautoshutdown',
stdoutf=None,
stderrf=None,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
log=True)
def _credential():
if os.path.exists('/tmp/cred-info'):
cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split(
'\t')
return {
'name':
'local',
'desc':
'Local credential',
'ctype':
ctype,
'cert':
open(cert).read(),
'pkey':
open(pkey).read(),
'metadata':
metadata
and dict([v.split('=', 1) for v in metadata.split(',')]) or {},
'conf':
config.configFromStream(open('/tmp/machine.conf'), lazy=True)
}
else:
return {
'name': 'local',
'desc': 'Local credential',
'ctype': 'local',
'cert': None,
'pkey': None,
'metadata': {},
'conf': config.configFromMap({})
}
def __init__(self, conf):
self.conf = conf
self.persistManager = persist.ClusterPersistManager()
self.machineConf = config.configFromStream(open(conf('config.machine_conf')),
base=config.configFromEnv())
self.clusterLocks = lock_manager.LockManager()
self.unresponsiveClusters = {}
def _loadTag():
tagPath = _createTagPath(self.conf, tagName)
if not os.path.exists(tagPath) and not os.path.exists(tagPath +
'.phantom'):
return defer.fail(TagNotFoundError(tagName))
if os.path.exists(tagPath + '.phantom'):
phantom = config.configFromStream(open(tagPath + '.phantom'),
lazy=True)
else:
phantom = None
if os.path.exists(tagPath + '.metadata'):
metadata = json.loads(open(tagPath + '.metadata').read())
else:
metadata = {}
if os.path.exists(tagPath):
files = [
f.rstrip('\n') for f in open(tagPath).readlines()
if f.strip()
]
else:
files = []
return Tag(tagName=tagName,
files=files,
metadata=metadata,
phantom=phantom,
taskName=metadata.get('task_name'))
def __init__(self, conf):
self.conf = conf
self.persistManager = persist.ClusterPersistManager()
self.machineConf = config.configFromStream(open(
conf('config.machine_conf')),
base=config.configFromEnv())
self.clusterLocks = lock_manager.LockManager()
self.unresponsiveClusters = {}
def instantiateCredential(conf, cred):
if not conf('config_loaded', default=False):
conf = config.configFromMap({'config_loaded': True},
base=config.configFromStream(open(
conf('general.conf_file',
default=DEFAULT_CONFIG_FILE)),
base=conf))
return (conf, None)
def __init__(self, conf):
self.conf = conf
self.pipelinePersist = persist.PipelinePersistManager()
self.tagNotify = tag_notify_listener.TagNotifyListener()
self.machineconf = config.configFromStream(open(conf('config.machine_conf')),
base=config.configFromEnv())
self.pipelinesCache = pipelines_cache.PipelinesCache(self.machineconf, self.pipelinePersist, self.tagNotify)
self.pipelinesMonitor = pipeline_monitor.PipelineMonitorManager()
def loadTagFile(fname):
"""
Loads a tagfile, returns a config object of attributes
Also considering a .phantom type which would represent files that don't really exist. I think this makes sense
as you should be able to tarnsfer .phantom files around but .metadata's should be generated when you make a tag
Will explain more abou this in a wiki page somewhere...
"""
##
# Phantom filse are in a format that configFromStream can read. This is because phantom files
# are expected to be written and modified by humans. .metadata files on the other hand
# are just expected to be the produce of a machine storing information so uses json
if os.path.exists(fname + '.phantom'):
##
# Put everythin under phantom
# We want to do it lazily too since we will be adding
# data it can access later
phantom = configFromMap(
{
'phantom_tag':
True,
'phantom':
configToDict(
configFromStream(open(fname + '.phantom'), lazy=True))
},
lazy=True)
else:
phantom = configFromMap({})
##
# If the fname actually exists, open its meta data + files
# if the fname does not exist but the phantom does, return the phantom
# otherwise, throw an exception about missing the tagfile
if os.path.exists(fname):
if os.path.exists(fname + '.metadata'):
metadata = configFromMap(
{'metadata': json.loads(open(fname + '.metadata').read())},
phantom,
lazy=True)
else:
metadata = configFromMap({}, phantom)
return configFromMap(
{'files': [f.strip() for f in open(fname) if f.strip()]},
metadata,
lazy=True)
elif not os.path.exists(fname) and os.path.exists(fname + '.phantom'):
if os.path.exists(fname + '.metadata'):
metadata = configFromMap(
{'metadata': json.loads(open(fname + '.metadata').read())},
phantom,
lazy=True)
return metadata
else:
return phantom
else:
raise MissingTagFileError(fname)
def __init__(self, conf):
self.conf = conf
self.pipelinePersist = persist.PipelinePersistManager()
self.tagNotify = tag_notify_listener.TagNotifyListener()
self.machineconf = config.configFromStream(open(
conf('config.machine_conf')),
base=config.configFromEnv())
self.pipelinesCache = pipelines_cache.PipelinesCache(
self.machineconf, self.pipelinePersist, self.tagNotify)
self.pipelinesMonitor = pipeline_monitor.PipelineMonitorManager()
def createExecDataFile(conf, master, masterMachineConf):
"""
Creates a exec data file as the perl start_cluster works
This is very similar to createMasterDataFile, should be refactored a bit
"""
outName = os.path.join('/tmp', str(time.time()))
##
# Going to load the master machine.conf and modify node type
masterConf = config.configFromStream(open(masterMachineConf), lazy=True)
masterConf = config.configFromMap({'NODE_TYPE': EXEC_NODE},
masterConf,
lazy=True)
fout = open(outName, 'w')
fout.write('\n'.join([
k + '=' + str(v)
for k, v in config.configToDict(masterConf).iteritems()
]))
fout.close()
template = open(conf('cluster.exec_user_data_tmpl')).read()
clusterPrivateKey = open(conf('cluster.cluster_private_key')).read()
outf = []
runSingleProgramEx('ssh-keygen -y -f ' +
conf('cluster.cluster_private_key'),
outf.append,
None,
log=True)
if conf('general.ctype') == 'ec2':
template = template.replace('<TMPL_VAR NAME=MASTER_DNS>',
master['private_dns'])
else:
template = template.replace('<TMPL_VAR NAME=MASTER_DNS>',
master['public_dns'])
clusterPublicKey = ''.join(outf)
template = template.replace('<TMPL_VAR NAME=CLUSTER_PRIVATE_KEY>',
clusterPrivateKey)
template = template.replace('<TMPL_VAR NAME=CLUSTER_PUBLIC_KEY>',
clusterPublicKey)
template = template.replace('<TMPL_VAR NAME=MACHINE_CONF>',
open(outName).read().replace('${', '\\${'))
os.remove(outName)
outf = os.path.join(conf('general.secure_tmp'), 'exec_user_data.sh')
open(outf, 'w').write(template)
return outf
def createCluster(request):
persistManager = request.state.persistManager
baseConf = config.configFromStream(open('/tmp/machine.conf'))
cluster = persist.Cluster(
request.body['cluster_name'], request.body['user_name'],
request.body['cred_name'],
config.configFromMap(request.body['conf'], base=baseConf))
yield persistManager.saveCluster(cluster)
defer.returnValue(request)
def createCluster(request):
persistManager = request.state.persistManager
baseConf = config.configFromStream(open('/tmp/machine.conf'))
cluster = persist.Cluster(request.body['cluster_name'],
request.body['user_name'],
request.body['cred_name'],
config.configFromMap(request.body['conf'],
base=baseConf))
yield persistManager.saveCluster(cluster)
defer.returnValue(request)
def instantiateCredential(conf, cred):
if not conf('config_loaded', default=False):
conf = config.configFromMap({'config_loaded': True},
base=config.configFromStream(open(
conf('general.conf_file')),
base=conf))
certFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_cert.pem')
keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem')
if not os.path.exists(certFile) and not os.path.exists(keyFile):
tmpCertFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_cert-tmp.pem')
tmpKeyFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_key-tmp.pem')
if 'ec2_url' not in cred.metadata:
raise Exception('You must have an ec2_url')
parsedUrl = urlparse.urlparse(cred.metadata['ec2_url'])
if ':' not in parsedUrl.netloc:
raise Exception('Your URL must contain a port')
host, port = parsedUrl.netloc.split(':')
fout = open(tmpCertFile, 'w')
fout.write(cred.cert)
fout.close()
fout = open(tmpKeyFile, 'w')
fout.write(cred.pkey)
fout.close()
try:
commands.runSystemEx(' '.join([
'nimbusCerts2EC2.py', '--in-cert=' + tmpCertFile,
'--out-cert=' + certFile, '--in-key=' + tmpKeyFile,
'--out-key=' + keyFile, '--java-cert-dir=/tmp',
'--java-cert-host=' + host, '--java-cert-port=' + port
]) + ' > /dev/null 2>&1',
log=True)
commands.runSystemEx('chmod +r ' + keyFile)
finally:
os.unlink(tmpCertFile)
os.unlink(tmpKeyFile)
ec2Home = '/opt/ec2-api-tools-1.3-42584'
newCred = func.Record(
cert=certFile,
pkey=keyFile,
ec2Path=os.path.join(ec2Home, 'bin'),
env=dict(EC2_JVM_ARGS='-Djavax.net.ssl.trustStore=/tmp/jssecacerts',
EC2_HOME=ec2Home,
EC2_URL=cred.metadata['ec2_url']))
if os.path.exists(conf('cluster.cluster_private_key') + '.pub'):
pubKey = open(conf('cluster.cluster_private_key') +
'.pub').read().rstrip()
ec2_control.addKeypair(newCred,
'"' + conf('cluster.key') + '||' + pubKey + '"')
return (conf, newCred)
def runPipelineConfig(taskName, name, pipeline, conf, queue=None):
"""
Takes a config object representing a pipeline options, validates those options
in pipeline.OPTIONS and passes the results onto runPipelineWithConfig
"""
##
# Mocheezmo way to have it load a conf file. This will be removed in the future
tmpConfigName = os.path.join('/tmp', str(time.time()) + '.config')
options = list(pipeline.OPTIONS)
options.append(
('conf', '', '--conf', 'Conf file (DO NOT SPECIFY, FOR INTERNAL USE)',
const('/tmp/machine.conf')))
options.append((
'CONFIG_FILE', '-c', '--CONFIG_FILE',
'Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line',
const(tmpConfigName)))
##
# Load up machine.conf and apply it to our current config
conf = config.configFromConfig(conf,
config.configFromStream(
open('/tmp/machine.conf'),
config.configFromEnv()),
lazy=True)
vals = {}
for o in options:
vals[o[0]] = cli.applyOption(conf(o[0], default=None), o, conf)
conf = config.configFromMap(vals, conf)
##
# For some ergatis trickery we then need to output this config to a temp file so ergatis can pull variables from it
confDict = config.configToDict(conf)
confVals = {}
cv = [('.'.join(k.split('.')[:-1]), k.split('.')[-1], v)
for k, v in confDict.iteritems()]
for s, k, v in cv:
confVals.setdefault(s, {})[k] = v
fout = open(tmpConfigName, 'w')
for s, d in confVals.iteritems():
if s not in ['', 'env']:
fout.write('[' + s + ']\n')
for k, v in d.iteritems():
fout.write('%s=%s\n' % (k, str(v)))
fout.close()
return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
def runPipelineConfig(taskName, name, pipeline, conf, queue=None):
"""
Takes a config object representing a pipeline options, validates those options
in pipeline.OPTIONS and passes the results onto runPipelineWithConfig
"""
##
# Mocheezmo way to have it load a conf file. This will be removed in the future
tmpConfigName = os.path.join("/tmp", str(time.time()) + ".config")
options = list(pipeline.OPTIONS)
options.append(("conf", "", "--conf", "Conf file (DO NOT SPECIFY, FOR INTERNAL USE)", const("/tmp/machine.conf")))
options.append(
(
"CONFIG_FILE",
"-c",
"--CONFIG_FILE",
"Config file for the pipeline. Specify this if you do not want to specify options on the comamnd line",
const(tmpConfigName),
)
)
##
# Load up machine.conf and apply it to our current config
conf = config.configFromConfig(
conf, config.configFromStream(open("/tmp/machine.conf"), config.configFromEnv()), lazy=True
)
vals = {}
for o in options:
vals[o[0]] = cli.applyOption(conf(o[0], default=None), o, conf)
conf = config.configFromMap(vals, conf)
##
# For some ergatis trickery we then need to output this config to a temp file so ergatis can pull variables from it
confDict = config.configToDict(conf)
confVals = {}
cv = [(".".join(k.split(".")[:-1]), k.split(".")[-1], v) for k, v in confDict.iteritems()]
for s, k, v in cv:
confVals.setdefault(s, {})[k] = v
fout = open(tmpConfigName, "w")
for s, d in confVals.iteritems():
if s not in ["", "env"]:
fout.write("[" + s + "]\n")
for k, v in d.iteritems():
fout.write("%s=%s\n" % (k, str(v)))
fout.close()
return runPipelineWithConfig(taskName, name, pipeline, conf, queue)
def createCluster(request):
"""Instantiates a skeleton of the cluster that will be imported. This
cluster will later have the proper attributes and values populated from
the source cluster.
"""
persistManager = request.state.persistManager
baseConf = config.configFromStream(open('/tmp/machine.conf'))
cluster = persist.Cluster(request.body['dst_cluster'],
request.body['user_name'],
request.body['cred_name'],
config.configFromMap({'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'},
base=baseConf))
yield persistManager.saveCluster(cluster)
defer.returnValue(request)
def createExecDataFile(conf, master, masterMachineConf):
"""
Creates a exec data file as the perl start_cluster works
This is very similar to createMasterDataFile, should be refactored a bit
"""
outName = os.path.join('/tmp', str(time.time()))
##
# Going to load the master machine.conf and modify node type
masterConf = config.configFromStream(open(masterMachineConf), lazy=True)
masterConf = config.configFromMap({'NODE_TYPE': EXEC_NODE}, masterConf, lazy=True)
fout = open(outName, 'w')
fout.write('\n'.join([k + '=' + str(v) for k, v in config.configToDict(masterConf).iteritems()]))
fout.close()
template = open(conf('cluster.exec_user_data_tmpl')).read()
clusterPrivateKey = open(conf('cluster.cluster_private_key')).read()
outf = []
runSingleProgramEx('ssh-keygen -y -f ' + conf('cluster.cluster_private_key'),
outf.append,
None,
log=True)
if conf('general.ctype') == 'ec2':
template = template.replace('<TMPL_VAR NAME=MASTER_DNS>', master['private_dns'])
else:
template = template.replace('<TMPL_VAR NAME=MASTER_DNS>', master['public_dns'])
clusterPublicKey = ''.join(outf)
template = template.replace('<TMPL_VAR NAME=CLUSTER_PRIVATE_KEY>', clusterPrivateKey)
template = template.replace('<TMPL_VAR NAME=CLUSTER_PUBLIC_KEY>', clusterPublicKey)
template = template.replace('<TMPL_VAR NAME=MACHINE_CONF>', open(outName).read().replace('${', '\\${'))
os.remove(outName)
outf = os.path.join(conf('general.secure_tmp'), 'exec_user_data.sh')
open(outf, 'w').write(template)
return outf
def createCluster(request):
"""Instantiates a skeleton of the cluster that will be imported. This
cluster will later have the proper attributes and values populated from
the source cluster.
"""
persistManager = request.state.persistManager
baseConf = config.configFromStream(open('/tmp/machine.conf'))
cluster = persist.Cluster(
request.body['dst_cluster'], request.body['user_name'],
request.body['cred_name'],
config.configFromMap(
{'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'},
base=baseConf))
yield persistManager.saveCluster(cluster)
defer.returnValue(request)
def _credential():
if os.path.exists('/tmp/cred-info'):
cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split('\t')
return {'name': 'local',
'desc': 'Local credential',
'ctype': ctype,
'cert': open(cert).read(),
'pkey': open(pkey).read(),
'metadata': metadata and dict([v.split('=', 1)
for v in metadata.split(',')]) or {},
'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True)}
else:
return {'name': 'local',
'desc': 'Local credential',
'ctype': 'local',
'cert': None,
'pkey': None,
'metadata': {},
'conf': config.configFromMap({})}
def instantiateCredential(conf, cred):
if not conf('config_loaded', default=False):
conf = config.configFromMap({'config_loaded': True},
base=config.configFromStream(open(conf('general.conf_file')), base=conf))
certFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert.pem')
keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem')
if not os.path.exists(certFile) and not os.path.exists(keyFile):
tmpCertFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert-tmp.pem')
tmpKeyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key-tmp.pem')
if 'ec2_url' not in cred.metadata:
raise Exception('You must have an ec2_url')
parsedUrl = urlparse.urlparse(cred.metadata['ec2_url'])
if ':' not in parsedUrl.netloc:
raise Exception('Your URL must contain a port')
host, port = parsedUrl.netloc.split(':')
fout = open(tmpCertFile, 'w')
fout.write(cred.cert)
fout.close()
fout = open(tmpKeyFile, 'w')
fout.write(cred.pkey)
fout.close()
try:
commands.runSystemEx(' '.join(['nimbusCerts2EC2.py',
'--in-cert=' + tmpCertFile,
'--out-cert=' + certFile,
'--in-key=' + tmpKeyFile,
'--out-key=' + keyFile,
'--java-cert-dir=/tmp',
'--java-cert-host=' + host,
'--java-cert-port=' + port]) + ' > /dev/null 2>&1', log=True)
commands.runSystemEx('chmod +r ' + keyFile)
finally:
os.unlink(tmpCertFile)
os.unlink(tmpKeyFile)
ec2Home = '/opt/ec2-api-tools-1.3-42584'
newCred = func.Record(cert=certFile, pkey=keyFile, ec2Path=os.path.join(ec2Home, 'bin'),
env=dict(EC2_JVM_ARGS='-Djavax.net.ssl.trustStore=/tmp/jssecacerts',
EC2_HOME=ec2Home,
EC2_URL=cred.metadata['ec2_url']))
if os.path.exists(conf('cluster.cluster_private_key') + '.pub'):
pubKey = open(conf('cluster.cluster_private_key') + '.pub').read().rstrip()
ec2_control.addKeypair(newCred, '"' + conf('cluster.key') + '||' + pubKey + '"')
return (conf, newCred)
def _delayAutoshutdown(state, batchState):
_log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file')
try:
clusters = yield clusters_client.listClusters('localhost', {
'cluster_name':
batchState['pipeline_config']['cluster.CLUSTER_NAME']
}, 'guest')
cluster = clusters[0]
if batchState.get('state', None) == COMPLETED_STATE:
_log(batchState, 'AUTOSHUTDOWN: Pipeline complete, done')
if batchState.get('state', None) != RUNNING_STATE:
_log(batchState,
'AUTOSHUTDOWN: Pipeline not running, calling later')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state,
batchState)
elif cluster['state'] == 'running':
# We need the SSH options from the machine.conf, ugh I hate these OOB dependencies
conf = config.configFromStream(open('/tmp/machine.conf'))
_log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown')
yield ssh.runProcessSSH(
cluster['master']['public_dns'],
'touch /var/vappio/runtime/delayautoshutdown',
stdoutf=None,
stderrf=None,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
log=True)
_log(batchState, 'AUTOSHUTDOWN: Setting up next call')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state,
batchState)
else:
_log(batchState,
'AUTOSHUTDOWN: Cluster not running, calling later')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state,
batchState)
except:
_log(batchState, 'AUTOSHUTDOWN: Cluster does not exist, calling later')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state,
batchState)
def _delayAutoshutdown(state, batchState):
_log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file')
if (('cluster_task' not in batchState
or batchState.get('state', None) != RUNNING_STATE)
and batchState.get('state', None) != COMPLETED_STATE):
# Not ready yet
_log(batchState,
'AUTOSHUTDOWN: No cluster or not running, calling later')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state,
batchState)
elif ('cluster_task' in batchState
and batchState.get('state', None) == RUNNING_STATE
and batchState.get('pipeline_state', None) != SHUTDOWN_STATE):
# Ready to see if resizing
_log(batchState, 'AUTOSHUTDOWN: Making sure cluster is up')
yield _blockOnTask(batchState['cluster_task'])
try:
cluster = yield loadCluster(
'localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest')
# We need the SSH options from the machine.conf, ugh I hate these OOB dependencies
conf = config.configFromStream(open('/tmp/machine.conf'))
_log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown')
yield ssh.runProcessSSH(
cluster['master']['public_dns'],
'touch /var/vappio/runtime/delayautoshutdown',
stdoutf=None,
stderrf=None,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
log=True)
except:
pass
_log(batchState, 'AUTOSHUTDOWN: Setting up next call')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state,
batchState)
else:
_log(batchState, 'AUTOSHUTDOWN: Giving up on this')
def loadTagFile(fname):
"""
Loads a tagfile, returns a config object of attributes
Also considering a .phantom type which would represent files that don't really exist. I think this makes sense
as you should be able to tarnsfer .phantom files around but .metadata's should be generated when you make a tag
Will explain more abou this in a wiki page somewhere...
"""
##
# Phantom filse are in a format that configFromStream can read. This is because phantom files
# are expected to be written and modified by humans. .metadata files on the other hand
# are just expected to be the produce of a machine storing information so uses json
if os.path.exists(fname + '.phantom'):
##
# Put everythin under phantom
# We want to do it lazily too since we will be adding
# data it can access later
phantom = configFromMap({'phantom_tag': True, 'phantom': configToDict(configFromStream(open(fname + '.phantom'), lazy=True))}, lazy=True)
else:
phantom = configFromMap({})
##
# If the fname actually exists, open its meta data + files
# if the fname does not exist but the phantom does, return the phantom
# otherwise, throw an exception about missing the tagfile
if os.path.exists(fname):
if os.path.exists(fname + '.metadata'):
metadata = configFromMap({'metadata': json.loads(open(fname + '.metadata').read())}, phantom, lazy=True)
else:
metadata = configFromMap({}, phantom)
return configFromMap({'files': [f.strip() for f in open(fname) if f.strip()]}, metadata, lazy=True)
elif not os.path.exists(fname) and os.path.exists(fname + '.phantom'):
if os.path.exists(fname + '.metadata'):
metadata = configFromMap({'metadata': json.loads(open(fname + '.metadata').read())}, phantom, lazy=True)
return metadata
else :
return phantom
else:
raise MissingTagFileError(fname)
def instantiateCredential(conf, cred):
"""
Takes a credential and instanitates it. It returns a Record that has all of the
information users of that instantiated credential will need
"""
if not conf("config_loaded", default=False):
conf = config.configFromMap(
{"config_loaded": True},
base=config.configFromStream(open(conf("general.conf_file", default=DEFAULT_CONFIG_FILE)), base=conf),
)
certFile = os.path.join(conf("general.secure_tmp"), cred.name + "_cert.pem")
keyFile = os.path.join(conf("general.secure_tmp"), cred.name + "_key.pem")
if not os.path.exists(certFile) or open(certFile).read() != cred.cert:
open(certFile, "w").write(cred.cert)
if not os.path.exists(keyFile) or open(keyFile).read() != cred.pkey:
open(keyFile, "w").write(cred.pkey)
newCred = functional.Record(cert=certFile, pkey=keyFile, ec2URL=None, env={})
if "ec2_url" in cred.metadata:
return (conf, newCred.update(env=functional.updateDict(newCred.env, dict(EC2_URL=cred.metadata["ec2_url"]))))
else:
return (conf, newCred)
def _loadTag():
tagPath = _createTagPath(self.conf, tagName)
if not os.path.exists(tagPath) and not os.path.exists(tagPath + ".phantom"):
return defer.fail(TagNotFoundError(tagName))
if os.path.exists(tagPath + ".phantom"):
phantom = config.configFromStream(open(tagPath + ".phantom"), lazy=True)
else:
phantom = None
if os.path.exists(tagPath + ".metadata"):
metadata = json.loads(open(tagPath + ".metadata").read())
else:
metadata = {}
if os.path.exists(tagPath):
files = [f.rstrip("\n") for f in open(tagPath).readlines() if f.strip()]
else:
files = []
return Tag(
tagName=tagName, files=files, metadata=metadata, phantom=phantom, taskName=metadata.get("task_name")
)
def _delayAutoshutdown(state, batchState):
_log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file')
if (('cluster_task' not in batchState or batchState.get('state', None) != RUNNING_STATE) and
batchState.get('state', None) != COMPLETED_STATE):
# Not ready yet
_log(batchState, 'AUTOSHUTDOWN: No cluster or not running, calling later')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
elif ('cluster_task' in batchState and
batchState.get('state', None) == RUNNING_STATE and
batchState.get('pipeline_state', None) != SHUTDOWN_STATE):
# Ready to see if resizing
_log(batchState, 'AUTOSHUTDOWN: Making sure cluster is up')
yield _blockOnTask(batchState['cluster_task'])
try:
cluster = yield loadCluster('localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
# We need the SSH options from the machine.conf, ugh I hate these OOB dependencies
conf = config.configFromStream(open('/tmp/machine.conf'))
_log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown')
yield ssh.runProcessSSH(cluster['master']['public_dns'],
'touch /var/vappio/runtime/delayautoshutdown',
stdoutf=None,
stderrf=None,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
log=True)
except:
pass
_log(batchState, 'AUTOSHUTDOWN: Setting up next call')
reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
else:
_log(batchState, 'AUTOSHUTDOWN: Giving up on this')
def _setQueue(taskName, batchState):
yield _blockOnTask(taskName)
cluster = yield loadCluster('localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
yield defer_utils.tryUntil(10,
lambda : _getOutput(batchState,
['/opt/clovr_pipelines/workflow/project_saved_templates/clovr_lgt_wrapper/set_queue.sh',
cluster['master']['public_dns']],
log=True),
onFailure=defer_utils.sleep(2))
conf = config.configFromStream(open('/tmp/machine.conf'))
# Remove autoshutdown, we want no part of that
yield ssh.runProcessSSH(cluster['master']['public_dns'],
'rm -v /var/vappio/runtime/noautoshutdown',
stdoutf=None,
stderrf=None,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
log=True)
def instantiateCredential(conf, cred):
if not conf('config_loaded', default=False):
conf = config.configFromConfig(conf,
base=config.configFromStream(open(
conf('conf_file')),
base=conf))
certFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_cert.pem')
keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem')
mainDeferred = defer.succeed(None)
if not os.path.exists(certFile) and not os.path.exists(keyFile):
tmpCertFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_cert-tmp.pem')
tmpKeyFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_key-tmp.pem')
if 'ec2_url' not in cred.metadata:
return defer.fail(Exception('You must have an ec2_url'))
parsedUrl = urlparse.urlparse(cred.metadata['ec2_url'])
if ':' not in parsedUrl.netloc:
return defer.fail(Exception('Your URL must contain a port'))
host, port = parsedUrl.netloc.split(':')
fout = open(tmpCertFile, 'w')
fout.write(cred.cert)
fout.close()
fout = open(tmpKeyFile, 'w')
fout.write(cred.pkey)
fout.close()
d = commands.runProcess([
'nimbusCerts2EC2.py', '--in-cert=' + tmpCertFile,
'--out-cert=' + certFile, '--in-key=' + tmpKeyFile,
'--out-key=' + keyFile, '--java-cert-dir=/tmp',
'--java-cert-host=' + host, '--java-cert-port=' + port
],
stdoutf=None,
stderrf=None,
log=True)
def _chmod(_exitCode):
return commands.runProcess(['chmod', '+r', keyFile],
stdoutf=None,
stderrf=None)
d.addCallback(_chmod)
def _unlink(v):
os.unlink(tmpCertFile)
os.unlink(tmpKeyFile)
return v
d.addCallback(_unlink)
d.addErrback(_unlink)
mainDeferred.addCallback(lambda _: d)
ec2Home = cred.metadata.get('ec2_api_tools',
'/opt/ec2-api-tools-1.3-57419')
newCred = func.Record(
name=cred.name,
conf=conf,
cert=certFile,
pkey=keyFile,
ec2Path=os.path.join(ec2Home, 'bin'),
env=dict(EC2_JVM_ARGS='-Djavax.net.ssl.trustStore=/tmp/jssecacerts',
EC2_HOME=ec2Home,
EC2_URL=cred.metadata['ec2_url']))
if os.path.exists(conf('cluster.cluster_private_key') + '.pub'):
pubKey = open(conf('cluster.cluster_private_key') +
'.pub').read().rstrip()
def _addKeypair():
keyPairDefer = ec2.addKeypair(newCred,
conf('cluster.key') + '||' + pubKey)
def _printError(f):
log.msg('Adding keypair failed, retrying')
log.err(f)
return f
keyPairDefer.addErrback(_printError)
return keyPairDefer
mainDeferred.addCallback(lambda _: defer_utils.tryUntil(
10, _addKeypair, onFailure=defer_utils.sleep(30)))
mainDeferred.addCallback(lambda _: newCred)
return mainDeferred
def _run(state, batchState):
if 'pipeline_name' in batchState:
pipelines = yield pipelines_client.pipelineList(
'localhost',
'local',
'guest',
batchState['pipeline_name'],
detail=True)
else:
pipelines = []
if not pipelines:
_log(batchState,
'First time running, creating pipeline state information')
batchState['pipeline_config'] = yield _applyActions(
state.innerPipelineConfig(), batchState['actions'])
batchState['pipeline_state'] = STARTCLUSTER_STATE
# We need to create a fake, local, pipeline for metrics to work
batchState['pipeline_name'] = pipeline_misc.checksumInput(
batchState['pipeline_config'])
batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState[
'pipeline_name']
batchState['pipeline_config'][
'pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name']
_log(batchState, 'Pipeline named ' + batchState['pipeline_name'])
pipeline = yield pipelines_client.createPipeline(
host='localhost',
clusterName='local',
userName='******',
pipelineName=batchState['pipeline_name'],
protocol='clovr_wrapper',
queue='pipeline.q',
config=batchState['pipeline_config'],
parentPipeline=state.parentPipeline())
batchState['lgt_wrapper_task_name'] = pipeline['task_name']
_log(
batchState,
'Setting number of tasks to 9 (number in a standard lgt_wrapper)')
yield _updateTask(batchState,
lambda t: t.update(completedTasks=0, numTasks=9))
state.updateBatchState()
batchState['state'] = RUNNING_STATE
_log(batchState,
'Pipeline started in %s state' % batchState['pipeline_state'])
yield _updateTask(batchState,
lambda t: t.setState(tasks.task.TASK_RUNNING))
pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs',
global_state.make_ref() + '.conf')
_log(batchState, 'Creating ergatis configuration')
_writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile)
if batchState['pipeline_state'] == STARTCLUSTER_STATE:
_log(batchState, 'Pipeline is in STARTCLUSTER state')
# First see if the cluster exists but is unresponsive
try:
cluster = yield loadCluster(
'localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest')
if cluster['state'] == 'unresponsive':
_log(batchState, 'Pipeline is unresponsive, terminating')
terminateTask = yield clusters_client.terminateCluster(
'localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
yield _blockOnTask(terminateTask)
except:
pass
batchState['cluster_task'] = yield startCluster(
batchState, 'localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest',
int(batchState['pipeline_config']['cluster.EXEC_NODES']), 0,
batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'], {
'cluster.MASTER_INSTANCE_TYPE':
batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'],
'cluster.MASTER_BID_PRICE':
batchState['pipeline_config']['cluster.MASTER_BID_PRICE'],
'cluster.EXEC_INSTANCE_TYPE':
batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'],
'cluster.EXEC_BID_PRICE':
batchState['pipeline_config']['cluster.EXEC_BID_PRICE']
})
yield _updateTask(batchState,
lambda t: t.update(completedTasks=0, numTasks=9))
yield _updateTask(
batchState, lambda t: t.addMessage(
tasks.task.MSG_SILENT, 'Completed startcluster').progress())
batchState['pipeline_state'] = REMOTE_LOCAL_TRANSFER_STATE
state.updateBatchState()
if batchState['pipeline_state'] == REMOTE_LOCAL_TRANSFER_STATE:
_log(batchState, 'Pipeline is in REMOTE_LOCAL_TRANSFER')
_log(batchState, 'Making sure cluster is exists in some form')
cluster = yield clusters_client.loadCluster(
'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
if cluster['state'] == 'unresponsive':
_log(batchState,
'Pipeline is unresponsive, erroring and restarting')
raise Exception('Cluster is not responsive')
yield state.prerunQueue.addWithDeferred(_remoteLocalTransfer,
batchState)
yield _updateTask(
batchState, lambda t: t.addMessage(
tasks.task.MSG_SILENT, 'Completed remote_local_transfer').
progress())
batchState['pipeline_state'] = DECRYPT_STATE
state.updateBatchState()
if batchState['pipeline_state'] == DECRYPT_STATE:
_log(batchState, 'Pipeline is in DECRYPT')
cluster = yield loadCluster(
'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
tag = yield tags_client.loadTag(
'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest', _decryptTagName(batchState))
conf = config.configFromStream(open('/tmp/machine.conf'))
yield ssh.runProcessSSH(cluster['master']['public_dns'],
'mkdir -p /mnt/lgt_decrypt',
stdoutf=None,
stderrf=None,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
log=True)
yield rsync.rsyncTo(
cluster['master']['public_dns'],
batchState['pipeline_config']['params.DECRYPT_SCRIPT'],
'/mnt/',
options=conf('rsync.options'),
user=conf('rsync.user'),
log=True)
for f in tag['files']:
decryptCmd = ' '.join([
os.path.join(
'/mnt',
os.path.basename(batchState['pipeline_config']
['params.DECRYPT_SCRIPT'])), f,
'-out-dir', '/mnt/lgt_decrypt', '-remove-encrypted',
'-password',
batchState['pipeline_config']['params.DECRYPT_PASSWORD']
])
yield ssh.getOutput(cluster['master']['public_dns'],
decryptCmd,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
expected=[0, 253],
log=True)
tag = yield tags_client.tagData(
host='localhost',
clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'],
userName='******',
action='overwrite',
tagName=_decryptTagName(batchState),
files=['/mnt/lgt_decrypt'],
metadata={},
recursive=True,
expand=False,
compressDir=None)
_log(
batchState, 'Waiting for tagging of %s to complete - %s' %
(_decryptTagName(batchState), tag['task_name']))
yield _blockOnTask(
tag['task_name'],
cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME'])
yield _updateTask(
batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT,
'Completed decrypt').progress())
batchState['pipeline_state'] = REFERENCE_TRANSFER_STATE
state.updateBatchState()
if batchState['pipeline_state'] == REFERENCE_TRANSFER_STATE:
_log(batchState, 'Pipeline is in REFERENCE_TRANSFER state')
transfers = []
tags = (batchState['pipeline_config']['input.REF_TAG1'].split(',') +
batchState['pipeline_config']['input.REF_TAG2'].split(','))
for tag in tags:
tag = tag.strip()
output = yield _getOutput(batchState, [
'vp-transfer-dataset', '-t', '--tag-name=' + tag,
'--dst-cluster=' +
batchState['pipeline_config']['cluster.CLUSTER_NAME']
],
log=True)
transfers.append(output['stdout'].strip())
for task in transfers:
yield _blockOnTask(task)
yield _updateTask(
batchState,
lambda t: t.addMessage(tasks.task.MSG_SILENT,
'Completed reference_transfer').progress())
batchState['pipeline_state'] = RUN_PIPELINE_STATE
state.updateBatchState()
if batchState['pipeline_state'] == RUN_PIPELINE_STATE:
_log(batchState, 'Pipeline is in RUN_PIPELINE state')
batchState['pipeline_config']['input.INPUT_TAG'] = _decryptTagName(
batchState)
pipeline = yield pipelines_client.runPipeline(
host='localhost',
clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'],
userName='******',
parentPipeline=batchState['pipeline_name'],
bareRun=True,
queue=state.innerPipelineQueue(),
config=batchState['pipeline_config'],
overwrite=True)
batchState['pipeline_task'] = pipeline['task_name']
yield _updateTask(
batchState, lambda t: t.addMessage(
tasks.task.MSG_SILENT, 'Completed run pipeline').progress())
batchState['pipeline_state'] = RUNNING_PIPELINE_STATE
state.updateBatchState()
if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE:
_log(batchState, 'Pipeline is in RUNNING_PIPELINE state')
_monitorPipeline(batchState)
yield _waitForPipeline(batchState)
yield _updateTask(
batchState,
lambda t: t.addMessage(tasks.task.MSG_SILENT,
'Completed running pipeline').progress())
batchState['pipeline_state'] = HARVEST_STATE
state.updateBatchState()
if batchState['pipeline_state'] == HARVEST_STATE:
_log(batchState, 'Pipeline is in HARVEST state')
# Using prerunqueue because we want everything here serialized
yield state.prerunQueue.addWithDeferred(_harvestTransfer, batchState)
yield _updateTask(
batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT,
'Completed harvest').progress())
batchState['pipeline_state'] = SHUTDOWN_STATE
state.updateBatchState()
if batchState['pipeline_state'] == SHUTDOWN_STATE:
_log(batchState, 'Pipeline is in SHUTDOWN state')
if 'add_instances_task' in batchState:
try:
yield _blockOnTask(batchState['add_instances_task'],
cluster=batchState['pipeline_config']
['cluster.CLUSTER_NAME'])
except Exception, err:
logging.errorPrint(str(err))
log.err(err)
yield clusters_client.terminateCluster(
'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
yield _updateTask(
batchState, lambda t: t.addMessage(
tasks.task.MSG_SILENT, 'Completed shutdown').progress())
batchState['pipeline_state'] = COMPLETED_STATE
batchState['state'] = COMPLETED_STATE
state.updateBatchState()
def instantiateCredential(conf, cred):
if not conf('config_loaded', default=False):
conf = config.configFromMap({'config_loaded': True},
base=config.configFromStream(open(conf('general.conf_file', default=DEFAULT_CONFIG_FILE)), base=conf))
return (conf, None)
from twisted.application import service
from twisted.python import log
from igs.utils import config
from vappio_tx.mq import client
from vappio_tx.internal_client import credentials
conf = config.configFromStream(open('/mnt/vappio-conf/vappio_apps.conf'))
application = service.Application('test')
s1 = client.makeService(conf)
s1.setServiceParent(application)
cc = credentials.CredentialClient('diag', s1.mqFactory, conf)
d = cc.listInstances()
def _terminate(instances):
print 'Num instances:', len(instances)
instances = instances[:3]
print 'Shutting down: ', instances
return cc.terminateInstances(instances)
def _print(foo):
print 'Foo:', foo
def buildConfigN(options,
args=None,
usage=None,
baseConf=None,
putInGeneral=True):
"""
This builds a config from options. Options is a list of tuples that looks like:
(name, short, long, help, func, [bool])
Where
name - Name of the option, this is what it will become in the config file
short - Short option - needs to start with -
long - Long option - needs to start with --
help - Help to be given to a user in --help output
func - Function to be applied to the value
bool - This is not required, set to True if the option is simply a boolean, all other datatypes can be verified via 'func'
This will implicitly check if a 'conf' option exists, and if so load te conf file as a base for these config options.
All options are put into the 'general' section.
This returns a tuple
(conf, args)
where args is whatever is left over from parsing
This also implicitly loads the current environment into the env section
If, when evaluated, 'func' returns a function, it is called with the baseConf. This is to allow more complex replacements to
happen.
"""
def _iterBool(v):
"""
Adds the non erquired bool field with a default of STRING if
it is not present
"""
for l in v:
if len(l) == 6:
yield l
else:
yield tuple(list(l) + [STRING])
parser = optparse.OptionParser(usage=usage)
##
# keep track of the function to apply to conf
confFunc = None
##
# The order of the options below
# var name, short option, long option, help message, function to apply, binary option
for n, s, l, h, f, b in _iterBool(options):
##
# We could have a function we want to apply to the conf variable. We want to store it
# so when we use it in the next block we don't have to loop over options looking for it again
# This is a minor optimization and probably not even necessary...
if n == 'conf':
confFunc = f
if b == BINARY:
parser.add_option(s, l, dest=n, help=h, action='store_true')
elif b == LIST:
parser.add_option(s, l, dest=n, help=h, action='append')
elif b == COUNT:
parser.add_option(s, l, dest=n, help=h, action='count')
elif b == STRING:
parser.add_option(s, l, dest=n, help=h)
else:
raise Exception('Unknown option type: ' + repr(b))
ops, args = parser.parse_args(args=args)
if baseConf is None:
baseConf = configFromEnv()
if hasattr(ops, 'conf'):
baseConf = configFromStream(
open(replaceStr(confFunc(ops.conf), baseConf)), baseConf)
vals = {}
##
# The order of the options below
# var name, short option, long option, help message, function to apply, binary option
for o in _iterBool(options):
n, _s, l, _h, f, _b = o
try:
vals[n] = applyOption(getattr(ops, n), o, baseConf)
except Exception, err:
raise CLIError(l, err)
from twisted.application import service
from twisted.python import log
from igs.utils import config
from vappio_tx.mq import client
from vappio_tx.internal_client import credentials
conf = config.configFromStream(open('/mnt/vappio-conf/vappio_apps.conf'))
application = service.Application('test')
s1 = client.makeService(conf)
s1.setServiceParent(application)
cc = credentials.CredentialClient('diag', s1.mqFactory, conf)
d = cc.listInstances()
def _terminate(instances):
print 'Num instances:', len(instances)
instances = instances[:3]
print 'Shutting down: ', instances
return cc.terminateInstances(instances)
def _print(foo):
print 'Foo:', foo
d.addCallback(_terminate)
d.addCallback(_print)
def instantiateCredential(conf, cred):
if not conf('config_loaded', default=False):
conf = config.configFromConfig(conf,
base=config.configFromStream(open(conf('conf_file')),
base=conf))
certFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert.pem')
keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem')
mainDeferred = defer.succeed(None)
if not os.path.exists(certFile) and not os.path.exists(keyFile):
tmpCertFile = os.path.join(conf('general.secure_tmp'), cred.name + '_cert-tmp.pem')
tmpKeyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key-tmp.pem')
if 'ec2_url' not in cred.metadata:
return defer.fail(Exception('You must have an ec2_url'))
parsedUrl = urlparse.urlparse(cred.metadata['ec2_url'])
if ':' not in parsedUrl.netloc:
return defer.fail(Exception('Your URL must contain a port'))
host, port = parsedUrl.netloc.split(':')
fout = open(tmpCertFile, 'w')
fout.write(cred.cert)
fout.close()
fout = open(tmpKeyFile, 'w')
fout.write(cred.pkey)
fout.close()
d = commands.runProcess(['nimbusCerts2EC2.py',
'--in-cert=' + tmpCertFile,
'--out-cert=' + certFile,
'--in-key=' + tmpKeyFile,
'--out-key=' + keyFile,
'--java-cert-dir=/tmp',
'--java-cert-host=' + host,
'--java-cert-port=' + port],
stdoutf=None,
stderrf=None,
log=True)
def _chmod(_exitCode):
return commands.runProcess(['chmod', '+r', keyFile], stdoutf=None, stderrf=None)
d.addCallback(_chmod)
def _unlink(v):
os.unlink(tmpCertFile)
os.unlink(tmpKeyFile)
return v
d.addCallback(_unlink)
d.addErrback(_unlink)
mainDeferred.addCallback(lambda _ : d)
ec2Home = cred.metadata.get('ec2_api_tools', '/opt/ec2-api-tools-1.3-57419')
newCred = func.Record(name=cred.name, conf=conf, cert=certFile, pkey=keyFile, ec2Path=os.path.join(ec2Home, 'bin'),
env=dict(EC2_JVM_ARGS='-Djavax.net.ssl.trustStore=/tmp/jssecacerts',
EC2_HOME=ec2Home,
EC2_URL=cred.metadata['ec2_url']))
if os.path.exists(conf('cluster.cluster_private_key') + '.pub'):
pubKey = open(conf('cluster.cluster_private_key') + '.pub').read().rstrip()
def _addKeypair():
keyPairDefer = ec2.addKeypair(newCred, conf('cluster.key') + '||' + pubKey)
def _printError(f):
log.msg('Adding keypair failed, retrying')
log.err(f)
return f
keyPairDefer.addErrback(_printError)
return keyPairDefer
mainDeferred.addCallback(lambda _ : defer_utils.tryUntil(10, _addKeypair, onFailure=defer_utils.sleep(30)))
mainDeferred.addCallback(lambda _ : newCred)
return mainDeferred
def buildConfigN(options, args=None, usage=None, baseConf=None, putInGeneral=True):
"""
This builds a config from options. Options is a list of tuples that looks like:
(name, short, long, help, func, [bool])
Where
name - Name of the option, this is what it will become in the config file
short - Short option - needs to start with -
long - Long option - needs to start with --
help - Help to be given to a user in --help output
func - Function to be applied to the value
bool - This is not required, set to True if the option is simply a boolean, all other datatypes can be verified via 'func'
This will implicitly check if a 'conf' option exists, and if so load te conf file as a base for these config options.
All options are put into the 'general' section.
This returns a tuple
(conf, args)
where args is whatever is left over from parsing
This also implicitly loads the current environment into the env section
If, when evaluated, 'func' returns a function, it is called with the baseConf. This is to allow more complex replacements to
happen.
"""
def _iterBool(v):
"""
Adds the non erquired bool field with a default of STRING if
it is not present
"""
for l in v:
if len(l) == 6:
yield l
else:
yield tuple(list(l) + [STRING])
parser = optparse.OptionParser(usage=usage)
##
# keep track of the function to apply to conf
confFunc = None
##
# The order of the options below
# var name, short option, long option, help message, function to apply, binary option
for n, s, l, h, f, b in _iterBool(options):
##
# We could have a function we want to apply to the conf variable. We want to store it
# so when we use it in the next block we don't have to loop over options looking for it again
# This is a minor optimization and probably not even necessary...
if n == 'conf':
confFunc = f
if b == BINARY:
parser.add_option(s, l, dest=n, help=h, action='store_true')
elif b == LIST:
parser.add_option(s, l, dest=n, help=h, action='append')
elif b == COUNT:
parser.add_option(s, l, dest=n, help=h, action='count')
elif b == STRING:
parser.add_option(s, l, dest=n, help=h)
else:
raise Exception('Unknown option type: ' + repr(b))
ops, args = parser.parse_args(args=args)
if baseConf is None:
baseConf = configFromEnv()
if hasattr(ops, 'conf'):
baseConf = configFromStream(open(replaceStr(confFunc(ops.conf), baseConf)), baseConf)
vals = {}
##
# The order of the options below
# var name, short option, long option, help message, function to apply, binary option
for o in _iterBool(options):
n, _s, l, _h, f, _b = o
try:
vals[n] = applyOption(getattr(ops, n), o, baseConf)
except Exception, err:
raise CLIError(l, err)
##
# These define useful policies for ensuring a machine is configed properly. This should probably be off lifted into
# a program/library designed for this, but what I need right now is simple
import os
from igs.utils.logging import errorPrintS
from igs.utils.commands import runSingleProgram, ProgramRunError
from igs.utils.config import configFromMap, configFromStream, configFromEnv, replaceStr
##
# These are default config options, these will be moved to a config file eventually
conf = configFromStream(open('/tmp/machine.conf'), configFromMap({
'stow': {'package_dir': '/usr/local/stow',
'base_dir': '/usr/local'},
'opt': {'package_dir': '/opt/opt-packages',
'base_dir': '/opt'},
'config': {'filename': '/tmp/machine.conf'},
}, configFromEnv()))
##
# Exceptions
class PolicyError(Exception):
pass
##
# A little helper function
def runSystemEx(cmd):
def configFromStream(stream):
return fixVariables(config.configFromStream(stream, config.configFromEnv()))
def loadLocalCluster(mq, state):
"""
If local cluster is not present, load it
"""
def _credential():
if os.path.exists('/tmp/cred-info'):
cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split('\t')
return {'name': 'local',
'desc': 'Local credential',
'ctype': ctype,
'cert': open(cert).read(),
'pkey': open(pkey).read(),
'metadata': metadata and dict([v.split('=', 1)
for v in metadata.split(',')]) or {},
'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True)}
else:
return {'name': 'local',
'desc': 'Local credential',
'ctype': 'local',
'cert': None,
'pkey': None,
'metadata': {},
'conf': config.configFromMap({})}
try:
cluster = yield state.persistManager.loadCluster('local', None)
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap({'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'},
base=baseConf)
if (cluster.credName == 'local' and
conf('MASTER_IP') not in [cluster.master['public_dns'],
cluster.master['private_dns']]):
master = dict(instance_id='local',
ami_id=None,
public_dns=conf('MASTER_IP'),
private_dns=conf('MASTER_IP'),
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master).update(config=conf)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
except persist.ClusterNotFoundError:
credential = _credential()
credTaskName = yield cred_client.saveCredential(credential['name'],
credential['desc'],
credential['ctype'],
credential['cert'],
credential['pkey'],
credential['metadata'],
credential['conf'])
## Wait for credential to be added.
## TODO: Should handle failure here
yield tasks_tx.blockOnTask('localhost',
'local',
credTaskName)
credClient = cred_client.CredentialClient('local',
mq,
state.conf)
## If it isn't a local ctype then we need to wait for
## the credential to come alive
if credential['ctype'] != 'local':
instances = yield credClient.listInstances()
else:
instances = []
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap({'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'},
base=baseConf)
cluster = persist.Cluster('local',
None,
'local',
conf)
startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1)
yield tasks_tx.updateTask(startTaskName,
lambda t : t.setState(tasks_tx.task.TASK_COMPLETED).progress())
cluster = cluster.update(startTask=startTaskName)
masterIp = cluster.config('MASTER_IP')
masterIdx = func.find(lambda i : masterIp in [i['public_dns'], i['private_dns']],
instances)
if masterIdx is not None:
master = instances[masterIdx]
else:
master = dict(instance_id='local',
ami_id=None,
public_dns=masterIp,
private_dns=masterIp,
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master)
cluster = cluster.setState(cluster.RUNNING)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
def _run(state, batchState):
if 'pipeline_name' in batchState:
pipelines = yield pipelines_client.pipelineList('localhost',
'local',
'guest',
batchState['pipeline_name'],
detail=True)
else:
pipelines = []
if not pipelines:
_log(batchState, 'First time running, creating pipeline state information')
batchState['pipeline_config'] = yield _applyActions(state.innerPipelineConfig(),
batchState['actions'])
batchState['pipeline_state'] = STARTCLUSTER_STATE
# We need to create a fake, local, pipeline for metrics to work
batchState['pipeline_name'] = pipeline_misc.checksumInput(batchState['pipeline_config'])
batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState['pipeline_name']
batchState['pipeline_config']['pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name']
_log(batchState, 'Pipeline named ' + batchState['pipeline_name'])
pipeline = yield pipelines_client.createPipeline(host='localhost',
clusterName='local',
userName='******',
pipelineName=batchState['pipeline_name'],
protocol='clovr_wrapper',
queue='pipeline.q',
config=batchState['pipeline_config'],
parentPipeline=state.parentPipeline())
batchState['lgt_wrapper_task_name'] = pipeline['task_name']
_log(batchState, 'Setting number of tasks to 9 (number in a standard lgt_wrapper)')
yield _updateTask(batchState,
lambda t : t.update(completedTasks=0,
numTasks=9))
state.updateBatchState()
batchState['state'] = RUNNING_STATE
_log(batchState, 'Pipeline started in %s state' % batchState['pipeline_state'])
yield _updateTask(batchState,
lambda t : t.setState(tasks.task.TASK_RUNNING))
pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs', global_state.make_ref() + '.conf')
_log(batchState, 'Creating ergatis configuration')
_writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile)
if batchState['pipeline_state'] == STARTCLUSTER_STATE:
_log(batchState, 'Pipeline is in STARTCLUSTER state')
# First see if the cluster exists but is unresponsive
try:
cluster = yield loadCluster('localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
if cluster['state'] == 'unresponsive':
_log(batchState, 'Pipeline is unresponsive, terminating')
terminateTask = yield clusters_client.terminateCluster('localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
yield _blockOnTask(terminateTask)
except:
pass
batchState['cluster_task'] = yield startCluster(
batchState,
'localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest',
int(batchState['pipeline_config']['cluster.EXEC_NODES']),
0,
batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'],
{'cluster.MASTER_INSTANCE_TYPE':
batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'],
'cluster.MASTER_BID_PRICE':
batchState['pipeline_config']['cluster.MASTER_BID_PRICE'],
'cluster.EXEC_INSTANCE_TYPE':
batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'],
'cluster.EXEC_BID_PRICE':
batchState['pipeline_config']['cluster.EXEC_BID_PRICE']})
yield _updateTask(batchState,
lambda t : t.update(completedTasks=0,
numTasks=9))
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed startcluster'
).progress())
batchState['pipeline_state'] = REMOTE_LOCAL_TRANSFER_STATE
state.updateBatchState()
if batchState['pipeline_state'] == REMOTE_LOCAL_TRANSFER_STATE:
_log(batchState, 'Pipeline is in REMOTE_LOCAL_TRANSFER')
_log(batchState, 'Making sure cluster is exists in some form')
cluster = yield clusters_client.loadCluster('localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
if cluster['state'] == 'unresponsive':
_log(batchState, 'Pipeline is unresponsive, erroring and restarting')
raise Exception('Cluster is not responsive')
yield state.prerunQueue.addWithDeferred(_remoteLocalTransfer,
batchState)
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed remote_local_transfer'
).progress())
batchState['pipeline_state'] = DECRYPT_STATE
state.updateBatchState()
if batchState['pipeline_state'] == DECRYPT_STATE:
_log(batchState, 'Pipeline is in DECRYPT')
cluster = yield loadCluster('localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
tag = yield tags_client.loadTag('localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest',
_decryptTagName(batchState))
conf = config.configFromStream(open('/tmp/machine.conf'))
yield ssh.runProcessSSH(cluster['master']['public_dns'],
'mkdir -p /mnt/lgt_decrypt',
stdoutf=None,
stderrf=None,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
log=True)
yield rsync.rsyncTo(cluster['master']['public_dns'],
batchState['pipeline_config']['params.DECRYPT_SCRIPT'],
'/mnt/',
options=conf('rsync.options'),
user=conf('rsync.user'),
log=True)
for f in tag['files']:
decryptCmd = ' '.join([os.path.join('/mnt', os.path.basename(batchState['pipeline_config']['params.DECRYPT_SCRIPT'])),
f,
'-out-dir', '/mnt/lgt_decrypt',
'-remove-encrypted',
'-password', batchState['pipeline_config']['params.DECRYPT_PASSWORD']])
yield ssh.getOutput(cluster['master']['public_dns'],
decryptCmd,
sshUser=conf('ssh.user'),
sshFlags=conf('ssh.options'),
expected=[0, 253],
log=True)
tag = yield tags_client.tagData(host='localhost',
clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'],
userName='******',
action='overwrite',
tagName=_decryptTagName(batchState),
files=['/mnt/lgt_decrypt'],
metadata={},
recursive=True,
expand=False,
compressDir=None)
_log(batchState, 'Waiting for tagging of %s to complete - %s' % (_decryptTagName(batchState),
tag['task_name']))
yield _blockOnTask(tag['task_name'],
cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME'])
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed decrypt'
).progress())
batchState['pipeline_state'] = REFERENCE_TRANSFER_STATE
state.updateBatchState()
if batchState['pipeline_state'] == REFERENCE_TRANSFER_STATE:
_log(batchState, 'Pipeline is in REFERENCE_TRANSFER state')
transfers = []
tags = (batchState['pipeline_config']['input.REF_TAG1'].split(',') +
batchState['pipeline_config']['input.REF_TAG2'].split(','))
for tag in tags:
tag = tag.strip()
output = yield _getOutput(batchState,
['vp-transfer-dataset',
'-t',
'--tag-name=' + tag,
'--dst-cluster=' + batchState['pipeline_config']['cluster.CLUSTER_NAME']],
log=True)
transfers.append(output['stdout'].strip())
for task in transfers:
yield _blockOnTask(task)
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed reference_transfer'
).progress())
batchState['pipeline_state'] = RUN_PIPELINE_STATE
state.updateBatchState()
if batchState['pipeline_state'] == RUN_PIPELINE_STATE:
_log(batchState, 'Pipeline is in RUN_PIPELINE state')
batchState['pipeline_config']['input.INPUT_TAG'] = _decryptTagName(batchState)
pipeline = yield pipelines_client.runPipeline(host='localhost',
clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'],
userName='******',
parentPipeline=batchState['pipeline_name'],
bareRun=True,
queue=state.innerPipelineQueue(),
config=batchState['pipeline_config'],
overwrite=True)
batchState['pipeline_task'] = pipeline['task_name']
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed run pipeline'
).progress())
batchState['pipeline_state'] = RUNNING_PIPELINE_STATE
state.updateBatchState()
if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE:
_log(batchState, 'Pipeline is in RUNNING_PIPELINE state')
_monitorPipeline(batchState)
yield _waitForPipeline(batchState)
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed running pipeline'
).progress())
batchState['pipeline_state'] = HARVEST_STATE
state.updateBatchState()
if batchState['pipeline_state'] == HARVEST_STATE:
_log(batchState, 'Pipeline is in HARVEST state')
# Using prerunqueue because we want everything here serialized
yield state.prerunQueue.addWithDeferred(_harvestTransfer,
batchState)
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed harvest'
).progress())
batchState['pipeline_state'] = SHUTDOWN_STATE
state.updateBatchState()
if batchState['pipeline_state'] == SHUTDOWN_STATE:
_log(batchState, 'Pipeline is in SHUTDOWN state')
if 'add_instances_task' in batchState:
try:
yield _blockOnTask(batchState['add_instances_task'],
cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME'])
except Exception, err:
logging.errorPrint(str(err))
log.err(err)
yield clusters_client.terminateCluster('localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest')
yield _updateTask(batchState,
lambda t : t.addMessage(tasks.task.MSG_SILENT,
'Completed shutdown'
).progress())
batchState['pipeline_state'] = COMPLETED_STATE
batchState['state'] = COMPLETED_STATE
state.updateBatchState()
import os
from igs.utils.logging import errorPrintS
from igs.utils.commands import runSingleProgram, ProgramRunError
from igs.utils.config import configFromMap, configFromStream, configFromEnv, replaceStr
##
# These are default config options, these will be moved to a config file eventually
conf = configFromStream(
open('/tmp/machine.conf'),
configFromMap(
{
'stow': {
'package_dir': '/usr/local/stow',
'base_dir': '/usr/local'
},
'opt': {
'package_dir': '/opt/opt-packages',
'base_dir': '/opt'
},
'config': {
'filename': '/tmp/machine.conf'
},
}, configFromEnv()))
##
# Exceptions
class PolicyError(Exception):
pass
def configFromStream(stream):
return fixVariables(config.configFromStream(stream,
config.configFromEnv()))
def loadLocalCluster(mq, state):
"""
If local cluster is not present, load it
"""
def _credential():
if os.path.exists('/tmp/cred-info'):
cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split(
'\t')
return {
'name':
'local',
'desc':
'Local credential',
'ctype':
ctype,
'cert':
open(cert).read(),
'pkey':
open(pkey).read(),
'metadata':
metadata
and dict([v.split('=', 1) for v in metadata.split(',')]) or {},
'conf':
config.configFromStream(open('/tmp/machine.conf'), lazy=True)
}
else:
return {
'name': 'local',
'desc': 'Local credential',
'ctype': 'local',
'cert': None,
'pkey': None,
'metadata': {},
'conf': config.configFromMap({})
}
try:
cluster = yield state.persistManager.loadCluster('local', None)
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap(
{
'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'
},
base=baseConf)
if (cluster.credName == 'local' and conf('MASTER_IP') not in [
cluster.master['public_dns'], cluster.master['private_dns']
]):
master = dict(instance_id='local',
ami_id=None,
public_dns=conf('MASTER_IP'),
private_dns=conf('MASTER_IP'),
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master).update(config=conf)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
except persist.ClusterNotFoundError:
credential = _credential()
credTaskName = yield cred_client.saveCredential(
credential['name'], credential['desc'], credential['ctype'],
credential['cert'], credential['pkey'], credential['metadata'],
credential['conf'])
## Wait for credential to be added.
## TODO: Should handle failure here
yield tasks_tx.blockOnTask('localhost', 'local', credTaskName)
credClient = cred_client.CredentialClient('local', mq, state.conf)
## If it isn't a local ctype then we need to wait for
## the credential to come alive
if credential['ctype'] != 'local':
instances = yield credClient.listInstances()
else:
instances = []
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap(
{
'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'
},
base=baseConf)
cluster = persist.Cluster('local', None, 'local', conf)
startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1)
yield tasks_tx.updateTask(
startTaskName,
lambda t: t.setState(tasks_tx.task.TASK_COMPLETED).progress())
cluster = cluster.update(startTask=startTaskName)
masterIp = cluster.config('MASTER_IP')
masterIdx = func.find(
lambda i: masterIp in [i['public_dns'], i['private_dns']],
instances)
if masterIdx is not None:
master = instances[masterIdx]
else:
master = dict(instance_id='local',
ami_id=None,
public_dns=masterIp,
private_dns=masterIp,
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master)
cluster = cluster.setState(cluster.RUNNING)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
