def exceptionMsg(exn, request):
"""
Takes a message queue, a dictionary that must contain the key `return_queue`
and an exception, the exception will be dumped to json
"""
returnQueueError(request.mq, request.body['return_queue'], str(exn))
return defer_pipe.ret(request)
def exceptionMsg(exn, request):
"""
Takes a message queue, a dictionary that must contain the key `return_queue`
and an exception, the exception will be dumped to json
"""
returnQueueError(request.mq, request.body['return_queue'], str(exn))
return defer_pipe.ret(request)
def failureMsg(f, request):
"""
Takes a message queue, a dictionary that must contain the key `return_queue`
and a failure, the failure will be dumped to json
"""
returnQueueFailure(request.mq, request.body['return_queue'], f)
return defer_pipe.ret(request)
def failureMsg(f, request):
"""
Takes a message queue, a dictionary that must contain the key `return_queue`
and a failure, the failure will be dumped to json
"""
returnQueueFailure(request.mq, request.body['return_queue'], f)
return defer_pipe.ret(request)
def _returnTag(request):
tag = persist.Tag(request.body['tag_name'],
[],
{},
None,
request.body['task_name'])
return defer_pipe.ret(request.update(response=persist.tagToDict(tag)))
def handleListInstances(request):
queue.returnQueueSuccess(request.mq,
request.body['return_queue'],
[request.credential.instanceToDict(i)
for i in request.state.credentialsCache.getCredential(request.credential.name)['instances']])
return defer_pipe.ret(request)
def handleCredentialConfig(request):
conf = config.configToDict(request.credential.credInstance.conf)
conf = func.updateDict(conf,
{'general.ctype': request.credential.credential.getCType()})
queue.returnQueueSuccess(request.mq, request.body['return_queue'], conf)
return defer_pipe.ret(request)
def handleCredentialConfig(request):
conf = config.configToDict(request.credential.credInstance.conf)
conf = func.updateDict(
conf, {'general.ctype': request.credential.credential.getCType()})
queue.returnQueueSuccess(request.mq, request.body['return_queue'], conf)
return defer_pipe.ret(request)
def wrapRequestHandler(state, f):
_addState = lambda r : defer_pipe.ret(r.update(state=state))
return defer_pipe.pipe([decodeRequest,
defer_pipe.hookError(defer_pipe.pipe([keysInBody(['return_queue']),
_addState,
f,
ackMsg]),
ackMsgFailure)])
def wrapRequestHandler(state, f):
_addState = lambda r: defer_pipe.ret(r.update(state=state))
return defer_pipe.pipe([
decodeRequest,
defer_pipe.hookError(
defer_pipe.pipe(
[keysInBody(['return_queue']), _addState, f, ackMsg]),
ackMsgFailure)
])
def handleUpdateInstances(request):
convertedInstances = [request.credential.instanceFromDict(i) for i in request.body["instances"]]
queue.returnQueueSuccess(
request.mq,
request.body["return_queue"],
[
request.credential.instanceToDict(ci)
for ci in request.state.credentialsCache.getCredential(request.credential.name)["instances"]
for i in convertedInstances
if (ci.spotRequestId and ci.spotRequestId == i.spotRequestId) or ci.instanceId == i.instanceId
],
)
return defer_pipe.ret(request)
def handleUpdateInstances(request):
convertedInstances = [
request.credential.instanceFromDict(i)
for i in request.body['instances']
]
queue.returnQueueSuccess(request.mq, request.body['return_queue'], [
request.credential.instanceToDict(ci)
for ci in request.state.credentialsCache.getCredential(
request.credential.name)['instances'] for i in convertedInstances
if (ci.spotRequestId and ci.spotRequestId == i.spotRequestId)
or ci.instanceId == i.instanceId
])
return defer_pipe.ret(request)
def handleWWWObserver(request):
"""
Input:
{ id: string
file: string
event: string
retval: string
props: string
host: string
time: string
name: string
message: string
}
Output:
None
"""
logging.debugPrint(lambda: repr(request.body))
request.mq.send('/queue/pipelines/observer/' + request.body['props'],
json.dumps(request.body))
return defer_pipe.ret(request.update(response=None))
def handleWWWObserver(request):
"""
Input:
{ id: string
file: string
event: string
retval: string
props: string
host: string
time: string
name: string
message: string
}
Output:
None
"""
logging.debugPrint(lambda : repr(request.body))
request.mq.send('/queue/pipelines/observer/' + request.body['props'],
json.dumps(request.body))
return defer_pipe.ret(request.update(response=None))
def decodeRequest((mq, msg)):
return defer_pipe.ret(QueueRequest(mq, msg))
def handleWWWListProtocols(request):
"""
Input:
{ cluster: string
?criteria: { key/value }
?detail: boolean
?batch_mode: boolean
}
Output:
[{ protocol: string, config: [ { key/value } ]}]
"""
def _removeAlwaysHidden(protocolConfig):
return [pc
for pc in protocolConfig
if pc[1].get('visibility') != 'always_hidden']
# Create a criterial filter called protocolF
if 'criteria' in request.body:
if '$or' in request.body['criteria']:
protocolNames = [p['protocol'] for p in request.body['criteria']['$or']]
protocolF = lambda p : p in protocolNames
elif 'protocol' in request.body['criteria']:
protocolF = lambda p : p == request.body['criteria']['protocol']
else:
protocolF = lambda _ : True
else:
protocolF = lambda _ : True
# Get all protocols and filter out the ones we don't need
protocols = [p
for p in protocol_format.protocols(request.state.machineconf)
if protocolF(p)]
if request.body.get('batch_mode'):
batchConfig = [func.updateDict(c[1], {'name': c[0]})
for c in _removeAlwaysHidden(protocol_format.load(request.state.machineconf,
'clovr_batch_wrapper'))]
else:
batchConfig = []
protocolConfs = []
for p in protocols:
protocolConfig = protocol_format.load(request.state.machineconf, p)
wrapperName = pipeline_misc.determineWrapper(request.state.machineconf, p)
if wrapperName != p:
protocolKeys = set([k for k, _ in protocolConfig])
wrapperConfig = [(k, v)
for (k,v) in protocol_format.load(request.state.machineconf,
wrapperName)
if k not in protocolKeys]
protocolConfig += wrapperConfig
if request.body.get('detail', False):
conf = [func.updateDict(c[1], {'name': c[0]})
for c in _removeAlwaysHidden(protocolConfig)]
if request.body.get('batch_mode', False):
conf = batchConfig + [func.updateDict(c, {'name': 'batch_pipeline.' + c['name']})
for c in conf]
else:
conf = []
protocolConfs.append({'protocol': p, 'config': conf})
return defer_pipe.ret(request.update(response=protocolConfs))
def _(request):
if core.keysInDict(ks, request.body):
return defer_pipe.ret(request)
else:
raise Exception('Missing keys in request %r %r' % (ks, request.body))
def returnSuccess(request):
returnQueueSuccess(request.mq, request.body['return_queue'], request.response)
return defer_pipe.ret(request)
def ackMsgFailure(f, request):
request.mq.ack(request.msg.headers['message-id'])
return defer_pipe.ret(request)
def ackMsgFailure(f, request):
request.mq.ack(request.msg.headers['message-id'])
return defer_pipe.ret(request)
def returnSuccess(request):
returnQueueSuccess(request.mq, request.body['return_queue'],
request.response)
return defer_pipe.ret(request)
def containsPipelineTemplate(request):
if 'pipeline.PIPELINE_TEMPLATE' in request.body['config']:
return defer_pipe.ret(request)
else:
raise Error('Must provide a config with pipeline.PIPELINE_TEMPLATE')
def _forward(taskName):
request.body['task_name'] = taskName
request.mq.send(dstQueue, json.dumps(request.body))
return defer_pipe.ret(request.update(response=taskName))
def _(request):
if core.keysInDict(ks, request.body):
return defer_pipe.ret(request)
else:
raise Exception('Missing keys in request %r %r' %
(ks, request.body))
def handleWWWListProtocols(request):
"""
Input:
{ cluster: string
?criteria: { key/value }
?detail: boolean
?batch_mode: boolean
}
Output:
[{ protocol: string, config: [ { key/value } ]}]
"""
def _removeAlwaysHidden(protocolConfig):
return [
pc for pc in protocolConfig
if pc[1].get('visibility') != 'always_hidden'
]
# Create a criterial filter called protocolF
if 'criteria' in request.body:
if '$or' in request.body['criteria']:
protocolNames = [
p['protocol'] for p in request.body['criteria']['$or']
]
protocolF = lambda p: p in protocolNames
elif 'protocol' in request.body['criteria']:
protocolF = lambda p: p == request.body['criteria']['protocol']
else:
protocolF = lambda _: True
else:
protocolF = lambda _: True
# Get all protocols and filter out the ones we don't need
protocols = [
p for p in protocol_format.protocols(request.state.machineconf)
if protocolF(p)
]
if request.body.get('batch_mode'):
batchConfig = [
func.updateDict(c[1], {'name': c[0]}) for c in _removeAlwaysHidden(
protocol_format.load(request.state.machineconf,
'clovr_batch_wrapper'))
]
else:
batchConfig = []
protocolConfs = []
for p in protocols:
protocolConfig = protocol_format.load(request.state.machineconf, p)
wrapperName = pipeline_misc.determineWrapper(request.state.machineconf,
p)
if wrapperName != p:
protocolKeys = set([k for k, _ in protocolConfig])
wrapperConfig = [(k, v) for (k, v) in protocol_format.load(
request.state.machineconf, wrapperName)
if k not in protocolKeys]
protocolConfig += wrapperConfig
if request.body.get('detail', False):
conf = [
func.updateDict(c[1], {'name': c[0]})
for c in _removeAlwaysHidden(protocolConfig)
]
if request.body.get('batch_mode', False):
conf = batchConfig + [
func.updateDict(c, {'name': 'batch_pipeline.' + c['name']})
for c in conf
]
else:
conf = []
protocolConfs.append({'protocol': p, 'config': conf})
return defer_pipe.ret(request.update(response=protocolConfs))
def handleGetCType(request):
queue.returnQueueSuccess(request.mq, request.body['return_queue'],
request.credential.credential.getCType())
return defer_pipe.ret(request)
def _returnTag(request):
tag = persist.Tag(request.body['tag_name'], [], {}, None,
request.body['task_name'])
return defer_pipe.ret(request.update(response=persist.tagToDict(tag)))
def decodeRequest((mq, msg)):
return defer_pipe.ret(QueueRequest(mq, msg))
def handleGetCType(request):
queue.returnQueueSuccess(request.mq,
request.body['return_queue'],
request.credential.credential.getCType())
return defer_pipe.ret(request)
def _forward(taskName):
request.body['task_name'] = taskName
request.mq.send(dstQueue, json.dumps(request.body))
return defer_pipe.ret(request.update(response=taskName))