def waitForInstances(instances, workflow):
"""
Takes instances and a workflow.
The input to the workflow is the current alive instances
and the output is alive and dead instances. Output
of entire function is alive and dead instances.
No instances are terminated in this.
"""
retInstances = func.Record(succeeded=instances, failed=[])
failedInstances = []
for f in workflow:
retInstances = yield f(retInstances.succeeded)
failedInstances.extend(retInstances.failed)
## If there aren't any instances left to check
## no reason to keep doing work.
if not retInstances.succeeded:
break
defer.returnValue(
func.Record(succeeded=retInstances.succeeded, failed=failedInstances))
def instantiateCredential(conf, cred):
"""
Takes a credential and instanitates it. It returns a Record that has all of the
information users of that instantiated credential will need
"""
if not conf('config_loaded', default=False):
conf = config.configFromMap({'config_loaded': True},
base=config.configFromStream(open(
conf('general.conf_file',
default=DEFAULT_CONFIG_FILE)),
base=conf))
certFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_cert.pem')
keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem')
if not os.path.exists(certFile) or open(certFile).read() != cred.cert:
open(certFile, 'w').write(cred.cert)
if not os.path.exists(keyFile) or open(keyFile).read() != cred.pkey:
open(keyFile, 'w').write(cred.pkey)
newCred = functional.Record(cert=certFile,
pkey=keyFile,
ec2URL=None,
env={})
if 'ec2_url' in cred.metadata:
return (conf,
newCred.update(env=functional.updateDict(
newCred.env, dict(EC2_URL=cred.metadata['ec2_url']))))
else:
return (conf, newCred)
def _monitorAnyPipelines(mq, state):
pipelines = yield state.pipelinePersist.loadAllPipelinesByAdmin({})
for p in pipelines:
if p.pipelineId:
yield pipeline_misc.monitorPipeline(
func.Record(state=state, mq=mq), p)
def instantiateCredential(conf, cred):
if 'conf_file' not in conf or not conf('conf_file'):
conf = config.configFromMap({'conf_file': DEFAULT_CONFIG_FILE},
base=conf)
newCred = func.Record(name=cred.name, conf=conf)
return defer.succeed(newCred)
def ensureInstances(f, instances, retries):
"""
Applies function f to each instance in instances. f
is a deferred that can evaluate to True, False, or
FAILED_INSTANCE
Returns a record with .succeeded and .failed
"""
retryInstances = []
succeeded = []
failed = []
while retries > 0 and instances:
yield defer_utils.sleep(30)()
for i in instances:
ret = yield f(i)
if ret == FAILED_INSTANCE:
failed.append(i)
elif ret:
succeeded.append(i)
else:
retryInstances.append(i)
retries -= 1
instances = retryInstances
retryInstances = []
if instances:
failed.extend(instances)
defer.returnValue(func.Record(succeeded=succeeded, failed=failed))
def _saveCluster(instances):
instances = yield credClient.updateInstances(instances)
cl = yield state.persistManager.loadCluster(cluster.clusterName,
cluster.userName)
cl = cl.addExecNodes(instances)
yield state.persistManager.saveCluster(cl)
defer.returnValue(func.Record(succeeded=instances, failed=[]))
def runThreadWithChannel(func):
"""
This creates a thread running the passed function. The only argument to the function
is the channel that will be used to communicate with the thread.
"""
ch = Channel()
th = runThread(func, ch)
return f.Record(thread=th, channel=ch)
def instantiateCredential(conf, cred):
"""Instantiates a credential based off the configuration provided."""
if 'conf_file' not in conf or not conf('conf_file'):
conf = config.configFromMap({'conf_file': DEFAULT_CONFIG_FILE},
base=conf)
newCred = func.Record(name=cred.name, conf=conf)
return defer.succeed(newCred)
def _updateTask(instances):
def _addMsg(t):
tt = t.addMessage(task.MSG_SILENT, msg)
tt = tt.addMessage(task.MSG_SILENT,
'Instances: %d' % len(instances))
tt = tt.progress()
return tt
yield tasks_tx.updateTask(taskName, _addMsg)
defer.returnValue(func.Record(succeeded=instances, failed=[]))
def instantiateCredential(conf, cred):
if not conf('config_loaded', default=False):
conf = config.configFromMap({'config_loaded': True},
base=config.configFromStream(open(
conf('general.conf_file')),
base=conf))
certFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_cert.pem')
keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem')
if not os.path.exists(certFile) and not os.path.exists(keyFile):
tmpCertFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_cert-tmp.pem')
tmpKeyFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_key-tmp.pem')
if 'ec2_url' not in cred.metadata:
raise Exception('You must have an ec2_url')
parsedUrl = urlparse.urlparse(cred.metadata['ec2_url'])
if ':' not in parsedUrl.netloc:
raise Exception('Your URL must contain a port')
host, port = parsedUrl.netloc.split(':')
fout = open(tmpCertFile, 'w')
fout.write(cred.cert)
fout.close()
fout = open(tmpKeyFile, 'w')
fout.write(cred.pkey)
fout.close()
try:
commands.runSystemEx(' '.join([
'nimbusCerts2EC2.py', '--in-cert=' + tmpCertFile,
'--out-cert=' + certFile, '--in-key=' + tmpKeyFile,
'--out-key=' + keyFile, '--java-cert-dir=/tmp',
'--java-cert-host=' + host, '--java-cert-port=' + port
]) + ' > /dev/null 2>&1',
log=True)
commands.runSystemEx('chmod +r ' + keyFile)
finally:
os.unlink(tmpCertFile)
os.unlink(tmpKeyFile)
ec2Home = '/opt/ec2-api-tools-1.3-42584'
newCred = func.Record(
cert=certFile,
pkey=keyFile,
ec2Path=os.path.join(ec2Home, 'bin'),
env=dict(EC2_JVM_ARGS='-Djavax.net.ssl.trustStore=/tmp/jssecacerts',
EC2_HOME=ec2Home,
EC2_URL=cred.metadata['ec2_url']))
if os.path.exists(conf('cluster.cluster_private_key') + '.pub'):
pubKey = open(conf('cluster.cluster_private_key') +
'.pub').read().rstrip()
ec2_control.addKeypair(newCred,
'"' + conf('cluster.key') + '||' + pubKey + '"')
return (conf, newCred)
def validatePipelineConfig(request):
validateState = func.Record(conf=request.state.conf,
machineconf=request.state.machineconf,
mq=request.mq)
protocolConf = protocol_format.load(request.state.machineconf,
request.body['config']['pipeline.PIPELINE_TEMPLATE'])
pipelineWrapper = determineWrapper(request.state.machineconf,
request.body['config']['pipeline.PIPELINE_TEMPLATE'])
if not request.body['bare_run']:
protocolConf += protocol_format.load(request.state.machineconf,
pipelineWrapper)
protocol_format.applyProtocol(protocolConf, request.body['config'])
return pipeline_validate.validate(validateState, protocolConf, request.body['config'])
def loadRepositories(configDir):
"""
Returns a map of repos names to record objects identifying the repos. Each record object contains:
- name
- rType (git, svn instantiated)
- repoUrl
- exportType
- branch
Everything but the logs is loaded here
"""
repos = {}
for line in (l for l in open(os.path.join(configDir, 'repositories'))
if l.strip()):
repoName, repoType, repoUrl = line.strip().split('\t')
repos[repoName] = func.Record(name=repoName,
rType=REPOSITORY_MAP[repoType](),
repoUrl=repoUrl,
exportType=EXPORT)
branches = []
for line in (l for l in open(os.path.join(configDir, 'branches'))
if l.strip()):
repoName, repoBranch = line.strip().split('\t')
repos[repoName] = repos[repoName].update(branch=repoBranch)
branches.append(repoName)
if set(branches) != set(repos.keys()):
raise Exception(
'You must have all the same repos in branches that you do in repositories'
)
try:
for line in (l for l in open(os.path.join(configDir, 'checkouts'))
if l.strip()):
repos[line.strip()] = repos[line.strip()].update(
exportType=CHECKOUT)
except IOError:
# File may not exist, ignore
pass
return repos
def runPipeline(request, pipeline):
runState = func.Record(conf=request.state.conf,
machineconf=request.state.machineconf,
mq=request.mq)
runningPipeline = yield pipeline_run.run(runState, pipeline)
pipelineMonitor = pipeline_monitor.PipelineMonitor(request.state,
request.mq,
runningPipeline,
int(request.state.conf('pipelines.retries')))
if request.state.pipelinesMonitor.contains(pipelineMonitor):
oldPipelineMonitor = request.state.pipelinesMonitor.remove(pipelineMonitor)
oldPipelineMonitor.release()
yield pipelineMonitor.initialize()
pipelineMonitor = request.state.pipelinesMonitor.add(pipelineMonitor)
taskRunPipeline = TaskRunPipeline(request.state.pipelinesCache, pipelineMonitor)
taskRunPipeline.initialize()
defer.returnValue(pipelineMonitor)
def validateBatchPipelineConfig(request):
"""
For batch configs we unfortunately cannot do as complete of
validation as we would like for the inner pipeline. In this case
we are going to create a validation scheme on the fly.
First we are going to pull out the inner pipeline config and
apply its protocol configuration to it. Then we are going to
modify the protocol configuration to match config names in
the request for the inner pipeline (prepending 'batch_pipeline.').
After that we are going to load the clovr_batch_wrapper protocol conf
and add it to the protocol conf we are building.
Then we need to modify the protocol conf to not try to validate things we
know will not validate.
"""
def _makeNoop(params):
# Turn off any validation if we are going to be replacing a variable
# in the variable
v['type'] = 'string'
v['require_value'] = False
# Remove any transform name so it doesn't accidently stomp on a variable
# we actually care about
if 'type_params' in v and 'transform_name' in v['type_params']:
v['type_params'].pop('transform_name')
# First things first, rip out the inner pipeline config:
innerConfig = dict([(k.split('.', 1)[-1], v)
for k, v in request.body['config'].iteritems()
if k.startswith('batch_pipeline.')])
protocolConf = protocol_format.load(request.state.machineconf,
innerConfig['pipeline.PIPELINE_TEMPLATE'])
protocol_format.applyProtocol(protocolConf, innerConfig)
# Push the applied protocol back into the request config
for k, v in innerConfig.iteritems():
request.body['config']['batch_pipeline.' + k] = v
additionalConf = []
for k, v in protocolConf:
if v['type'] in ['dataset', 'paired_dataset', 'singleton_dataset']:
if ('batch.tag_list.' + k) in request.body['config']:
params = dict(v)
params['type_params'] = dict(params.get('type_params', {}))
typeParams = params['type_params']
typeParams['transform_name'] = 'batch.tag_list.file_list.' + k + '_throwaway'
additionalConf.append(('batch.tag_list.' + k, params))
_makeNoop(v)
elif ('batch.param_list.' + k) in request.body['config']:
params = dict(v)
params['type'] = params['type'] + ' list'
params['type_params'] = dict(params.get('type_params', {}))
typeParams = params['type_params']
typeParams['transform_name'] = 'batch.param_list.file_list.' + k + '_throwaway'
additionalConf.append(('batch.param_list.' + k, params))
_makeNoop(v)
elif '${BATCH_NUM}' in request.body['config'].get(k, ''):
_makeNoop(v)
elif ('batch.param_list.' + k) in request.body['config']:
params = dict(v)
params['type'] = params['type'] + ' list'
params['type_params'] = dict(params.get('type_params', {}))
typeParams = params['type_params']
typeParams['transform_name'] = 'batch.param_list.transformed.' + k + '_throwaway'
elif '${BATCH_NUM}' in request.body['config'].get(k, ''):
_makeNoop(v)
else:
if 'type_params' in v and 'transform_name' in v['type_params']:
v['type_params']['transform_name'] = 'batch_pipeline.' + v['type_params']['transform_name']
protocolConf = [('batch_pipeline.' + k, v)
for k, v in protocolConf]
batchWrapperConf = protocol_format.load(request.state.machineconf,
'clovr_batch_wrapper')
protocol_format.applyProtocol(batchWrapperConf, request.body['config'])
protocolConf += batchWrapperConf
protocolConf += additionalConf
validateState = func.Record(conf=request.state.conf,
machineconf=request.state.machineconf,
mq=request.mq)
return pipeline_validate.validate(validateState, protocolConf, request.body['config'])
def instantiateCredential(conf, cred):
if not conf('config_loaded', default=False):
conf = config.configFromConfig(conf,
base=config.configFromStream(open(
conf('conf_file')),
base=conf))
certFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_cert.pem')
keyFile = os.path.join(conf('general.secure_tmp'), cred.name + '_key.pem')
mainDeferred = defer.succeed(None)
if not os.path.exists(certFile) and not os.path.exists(keyFile):
tmpCertFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_cert-tmp.pem')
tmpKeyFile = os.path.join(conf('general.secure_tmp'),
cred.name + '_key-tmp.pem')
if 'ec2_url' not in cred.metadata:
return defer.fail(Exception('You must have an ec2_url'))
parsedUrl = urlparse.urlparse(cred.metadata['ec2_url'])
if ':' not in parsedUrl.netloc:
return defer.fail(Exception('Your URL must contain a port'))
host, port = parsedUrl.netloc.split(':')
fout = open(tmpCertFile, 'w')
fout.write(cred.cert)
fout.close()
fout = open(tmpKeyFile, 'w')
fout.write(cred.pkey)
fout.close()
d = commands.runProcess([
'nimbusCerts2EC2.py', '--in-cert=' + tmpCertFile,
'--out-cert=' + certFile, '--in-key=' + tmpKeyFile,
'--out-key=' + keyFile, '--java-cert-dir=/tmp',
'--java-cert-host=' + host, '--java-cert-port=' + port
],
stdoutf=None,
stderrf=None,
log=True)
def _chmod(_exitCode):
return commands.runProcess(['chmod', '+r', keyFile],
stdoutf=None,
stderrf=None)
d.addCallback(_chmod)
def _unlink(v):
os.unlink(tmpCertFile)
os.unlink(tmpKeyFile)
return v
d.addCallback(_unlink)
d.addErrback(_unlink)
mainDeferred.addCallback(lambda _: d)
ec2Home = cred.metadata.get('ec2_api_tools',
'/opt/ec2-api-tools-1.3-57419')
newCred = func.Record(
name=cred.name,
conf=conf,
cert=certFile,
pkey=keyFile,
ec2Path=os.path.join(ec2Home, 'bin'),
env=dict(EC2_JVM_ARGS='-Djavax.net.ssl.trustStore=/tmp/jssecacerts',
EC2_HOME=ec2Home,
EC2_URL=cred.metadata['ec2_url']))
if os.path.exists(conf('cluster.cluster_private_key') + '.pub'):
pubKey = open(conf('cluster.cluster_private_key') +
'.pub').read().rstrip()
def _addKeypair():
keyPairDefer = ec2.addKeypair(newCred,
conf('cluster.key') + '||' + pubKey)
def _printError(f):
log.msg('Adding keypair failed, retrying')
log.err(f)
return f
keyPairDefer.addErrback(_printError)
return keyPairDefer
mainDeferred.addCallback(lambda _: defer_utils.tryUntil(
10, _addKeypair, onFailure=defer_utils.sleep(30)))
mainDeferred.addCallback(lambda _: newCred)
return mainDeferred
