def _createPipeline(request):
taskName = yield tasks_tx.createTaskAndSave('runPipelines', 0)
# The name of a pipeline is being stored as a checksum. Pipeline names
# are arbitrary and the user will likely never know or care what it is.
# The pipeline name still exists though because other tools will likely
# find it useful to refer to a pipeline by a particular name, but if
# we decide to change the pipeline name to something more meaningful they
# won't have to chagne their code to use pipelineName instead of checksum
protocol = _determineProtocol(request)
if not request.body['bare_run']:
request.body['config'][
'pipeline.PIPELINE_WRAPPER_NAME'] = request.body['config'][
'pipeline.PIPELINE_NAME']
defer.returnValue(
persist.Pipeline(pipelineId=None,
pipelineName=checksum,
userName=request.body['user_name'],
protocol=protocol,
checksum=checksum,
taskName=taskName,
queue=request.body.get('queue',
'pipelinewrapper.q'),
children=[],
config=request.body['config']))
def _(request):
def _forward(taskName):
request.body['task_name'] = taskName
request.mq.send(dstQueue, json.dumps(request.body))
return defer_pipe.ret(request.update(response=taskName))
d = tasks.createTaskAndSave(tType, numTasks)
d.addCallback(_forward)
return d
def handleWWWPipelineCreate(request):
"""
Sets the config section of a pipeline exactly to what is given
Input:
{ cluster: string
user_name: string
pipeline_name : string
protocol : string
queue : string
config: { key/value }
?pipeline_id : string
?task_name : string
?pipeline_parent : pipeline_name
}
Output:
Pipeline
"""
if request.body.get('parent_pipeline'):
parentPipelines = yield request.state.pipelinePersist.loadAllPipelinesBy({'pipeline_name': request.body['parent_pipeline']},
request.body['user_name'])
if len(parentPipelines) == 1:
parentPipeline = parentPipelines[0]
else:
raise Exception('More than one possible parent pipeline choice, not sure what to do here')
else:
parentPipeline = None
if 'task_name' in request.body:
taskName = request.body['task_name']
else:
taskName = yield tasks.createTaskAndSave('runPipelines', 0)
pipeline = persist.Pipeline(pipelineId=request.body.get('pipeline_id', None),
pipelineName=request.body['pipeline_name'],
userName=request.body['user_name'],
protocol=request.body['protocol'],
checksum=pipeline_misc.checksumInput(request.body['config']),
taskName=taskName,
queue=request.body['queue'],
children=[],
config=request.body['config'])
yield request.state.pipelinePersist.savePipeline(pipeline)
pipelineDict = yield request.state.pipelinesCache.pipelineToDict(pipeline)
if parentPipeline:
childPipeline = [('local',
request.body['pipeline_name'])]
parentPipeline = parentPipeline.update(children=list(set([tuple(e)
for e in parentPipeline.children + childPipeline])))
yield request.state.pipelinePersist.savePipeline(parentPipeline)
defer.returnValue(request.update(response=pipelineDict))
def handleWWWListAddCredentials(request):
if 'credential_name' in request.body and core.keysInDict(
['credential_name', 'description', 'ctype', 'metadata'], request.body):
# Users can provide a file name or the actual contents of the
# certificate.
if 'cert_file' in request.body:
cert = open(request.body['cert_file']).read()
else:
cert = request.body['cert']
if 'pkey_file' in request.body:
pkey = open(request.body['pkey_file']).read()
else:
pkey = request.body['pkey']
conf = config.configFromMap(request.body.get('conf', {}),
base=config.configFromEnv())
cred = persist.createCredential(name=request.body['credential_name'],
desc=request.body['description'],
ctype=request.body['ctype'],
cert=cert,
pkey=pkey,
active=True,
metadata=request.body['metadata'],
conf=conf)
taskName = yield tasks_tx.createTaskAndSave('addCredential', 1)
instantiateAndSaveCredential(taskName, cred,
request.state.credentialPersist)
queue.returnQueueSuccess(request.mq, request.body['return_queue'],
taskName)
defer.returnValue(request)
elif 'credential_name' not in request.body:
credentials = request.state.credentialsCache.getAllCredentials()
credentialsDicts = [{
'name': name,
'description': c['cred_instance'].credential.desc,
'num_instances': len(c['instances']),
'ctype': c['cred_instance'].credential.getCType()
} for name, c in credentials.iteritems()
if ('credential_names' in request.body
and name in request.body['credential_names'])
or 'credential_names' not in request.body]
queue.returnQueueSuccess(request.mq, request.body['return_queue'],
credentialsDicts)
defer.returnValue(request)
else:
queue.returnQueueError(request.mq, request.body['return_queue'],
'Unknown credential query')
raise UnknownRequestError(str(request.body))
def _(request):
def _forward(taskName):
request.body['task_name'] = taskName
request.mq.send(dstQueue, json.dumps(request.body))
return defer_pipe.ret(request.update(response=taskName))
d = tasks.createTaskAndSave(tType, numTasks)
d.addCallback(_forward)
return d
def _createPipeline(request):
taskName = yield tasks_tx.createTaskAndSave('runPipelines', 0)
# The name of a pipeline is being stored as a checksum. Pipeline names
# are arbitrary and the user will likely never know or care what it is.
# The pipeline name still exists though because other tools will likely
# find it useful to refer to a pipeline by a particular name, but if
# we decide to change the pipeline name to something more meaningful they
# won't have to chagne their code to use pipelineName instead of checksum
protocol = _determineProtocol(request)
if not request.body['bare_run']:
request.body['config']['pipeline.PIPELINE_WRAPPER_NAME'] = request.body['config']['pipeline.PIPELINE_NAME']
defer.returnValue(persist.Pipeline(pipelineId=None,
pipelineName=checksum,
userName=request.body['user_name'],
protocol=protocol,
checksum=checksum,
taskName=taskName,
queue=request.body.get('queue', 'pipelinewrapper.q'),
children=[],
config=request.body['config']))
def handleWWWListAddCredentials(request):
if 'credential_name' in request.body and core.keysInDict(['credential_name',
'description',
'ctype',
'metadata'],
request.body):
# Users can provide a file name or the actual contents of the
# certificate.
if 'cert_file' in request.body:
cert = open(request.body['cert_file']).read()
else:
cert = request.body['cert']
if 'pkey_file' in request.body:
pkey = open(request.body['pkey_file']).read()
else:
pkey = request.body['pkey']
conf = config.configFromMap(request.body.get('conf', {}),
base=config.configFromEnv())
cred = persist.createCredential(name=request.body['credential_name'],
desc=request.body['description'],
ctype=request.body['ctype'],
cert=cert,
pkey=pkey,
active=True,
metadata=request.body['metadata'],
conf=conf)
taskName = yield tasks_tx.createTaskAndSave('addCredential', 1)
instantiateAndSaveCredential(taskName, cred, request.state.credentialPersist)
queue.returnQueueSuccess(request.mq,
request.body['return_queue'],
taskName)
defer.returnValue(request)
elif 'credential_name' not in request.body:
credentials = request.state.credentialsCache.getAllCredentials()
credentialsDicts = [{'name': name,
'description': c['cred_instance'].credential.desc,
'num_instances': len(c['instances']),
'ctype': c['cred_instance'].credential.getCType()}
for name, c in credentials.iteritems()
if ('credential_names' in request.body and name in request.body['credential_names']) or
'credential_names' not in request.body]
queue.returnQueueSuccess(request.mq,
request.body['return_queue'],
credentialsDicts)
defer.returnValue(request)
else:
queue.returnQueueError(request.mq,
request.body['return_queue'],
'Unknown credential query')
raise UnknownRequestError(str(request.body))
def loadLocalCluster(mq, state):
"""
If local cluster is not present, load it
"""
def _credential():
if os.path.exists('/tmp/cred-info'):
cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split('\t')
return {'name': 'local',
'desc': 'Local credential',
'ctype': ctype,
'cert': open(cert).read(),
'pkey': open(pkey).read(),
'metadata': metadata and dict([v.split('=', 1)
for v in metadata.split(',')]) or {},
'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True)}
else:
return {'name': 'local',
'desc': 'Local credential',
'ctype': 'local',
'cert': None,
'pkey': None,
'metadata': {},
'conf': config.configFromMap({})}
try:
cluster = yield state.persistManager.loadCluster('local', None)
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap({'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'},
base=baseConf)
if (cluster.credName == 'local' and
conf('MASTER_IP') not in [cluster.master['public_dns'],
cluster.master['private_dns']]):
master = dict(instance_id='local',
ami_id=None,
public_dns=conf('MASTER_IP'),
private_dns=conf('MASTER_IP'),
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master).update(config=conf)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
except persist.ClusterNotFoundError:
credential = _credential()
credTaskName = yield cred_client.saveCredential(credential['name'],
credential['desc'],
credential['ctype'],
credential['cert'],
credential['pkey'],
credential['metadata'],
credential['conf'])
## Wait for credential to be added.
## TODO: Should handle failure here
yield tasks_tx.blockOnTask('localhost',
'local',
credTaskName)
credClient = cred_client.CredentialClient('local',
mq,
state.conf)
## If it isn't a local ctype then we need to wait for
## the credential to come alive
if credential['ctype'] != 'local':
instances = yield credClient.listInstances()
else:
instances = []
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap({'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'},
base=baseConf)
cluster = persist.Cluster('local',
None,
'local',
conf)
startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1)
yield tasks_tx.updateTask(startTaskName,
lambda t : t.setState(tasks_tx.task.TASK_COMPLETED).progress())
cluster = cluster.update(startTask=startTaskName)
masterIp = cluster.config('MASTER_IP')
masterIdx = func.find(lambda i : masterIp in [i['public_dns'], i['private_dns']],
instances)
if masterIdx is not None:
master = instances[masterIdx]
else:
master = dict(instance_id='local',
ami_id=None,
public_dns=masterIp,
private_dns=masterIp,
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master)
cluster = cluster.setState(cluster.RUNNING)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
def loadLocalCluster(mq, state):
"""
If local cluster is not present, load it
"""
def _credential():
if os.path.exists('/tmp/cred-info'):
cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split(
'\t')
return {
'name':
'local',
'desc':
'Local credential',
'ctype':
ctype,
'cert':
open(cert).read(),
'pkey':
open(pkey).read(),
'metadata':
metadata
and dict([v.split('=', 1) for v in metadata.split(',')]) or {},
'conf':
config.configFromStream(open('/tmp/machine.conf'), lazy=True)
}
else:
return {
'name': 'local',
'desc': 'Local credential',
'ctype': 'local',
'cert': None,
'pkey': None,
'metadata': {},
'conf': config.configFromMap({})
}
try:
cluster = yield state.persistManager.loadCluster('local', None)
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap(
{
'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'
},
base=baseConf)
if (cluster.credName == 'local' and conf('MASTER_IP') not in [
cluster.master['public_dns'], cluster.master['private_dns']
]):
master = dict(instance_id='local',
ami_id=None,
public_dns=conf('MASTER_IP'),
private_dns=conf('MASTER_IP'),
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master).update(config=conf)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
except persist.ClusterNotFoundError:
credential = _credential()
credTaskName = yield cred_client.saveCredential(
credential['name'], credential['desc'], credential['ctype'],
credential['cert'], credential['pkey'], credential['metadata'],
credential['conf'])
## Wait for credential to be added.
## TODO: Should handle failure here
yield tasks_tx.blockOnTask('localhost', 'local', credTaskName)
credClient = cred_client.CredentialClient('local', mq, state.conf)
## If it isn't a local ctype then we need to wait for
## the credential to come alive
if credential['ctype'] != 'local':
instances = yield credClient.listInstances()
else:
instances = []
baseConf = config.configFromStream(open('/tmp/machine.conf'),
base=config.configFromEnv())
conf = config.configFromMap(
{
'config_loaded': True,
'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'
},
base=baseConf)
cluster = persist.Cluster('local', None, 'local', conf)
startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1)
yield tasks_tx.updateTask(
startTaskName,
lambda t: t.setState(tasks_tx.task.TASK_COMPLETED).progress())
cluster = cluster.update(startTask=startTaskName)
masterIp = cluster.config('MASTER_IP')
masterIdx = func.find(
lambda i: masterIp in [i['public_dns'], i['private_dns']],
instances)
if masterIdx is not None:
master = instances[masterIdx]
else:
master = dict(instance_id='local',
ami_id=None,
public_dns=masterIp,
private_dns=masterIp,
state='running',
key=None,
index=None,
instance_type=None,
launch=None,
availability_zone=None,
monitor=None,
spot_request_id=None,
bid_price=None)
cluster = cluster.setMaster(master)
cluster = cluster.setState(cluster.RUNNING)
yield state.persistManager.saveCluster(cluster)
defer.returnValue(cluster)
