def makeService(conf):
mqService = client.makeService(conf)
mqFactory = mqService.mqFactory
processRequest = queue.returnResponse(
defer_pipe.pipe([
queue.keysInBody(['cluster', 'tasklet', 'conf']),
forwardOrCreate(
conf('www.url_prefix') + '/' +
os.path.basename(conf('tasklets.tasklets_www')),
conf('tasklets.tasklets_queue'), 'runTasklets', 1)
]))
queue.subscribe(mqFactory, conf('tasklets.tasklets_www'),
conf('tasklets.concurrent_tasklets'),
queue.wrapRequestHandler(None, processRequest))
processRunTasklet = defer_pipe.hookError(
defer_pipe.pipe(
[queue.keysInBody(['tasklet', 'conf']), handleRunTasklet]),
queue.failureMsg)
queue.subscribe(mqFactory, conf('tasklets.tasklets_queue'),
conf('tasklets.concurrent_tasklets'),
queue.wrapRequestHandlerTask(None, processRunTasklet))
return mqService
def subscribe(mq, state):
createAndForward = queue.createTaskAndForward(state.conf('clusters.startcluster_queue'),
'startCluster',
5)
processStartPipe = defer_pipe.pipe([queue.keysInBody(['cluster_name',
'user_name',
'num_exec',
'num_data',
'cred_name',
'conf']),
returnClusterStartTaskIfExists,
createCluster,
createAndForward])
processStartCluster = queue.returnResponse(processStartPipe)
queue.subscribe(mq,
state.conf('clusters.startcluster_www'),
state.conf('clusters.concurrent_startcluster'),
queue.wrapRequestHandler(state, processStartCluster))
queue.subscribe(mq,
state.conf('clusters.startcluster_queue'),
state.conf('clusters.concurrent_startcluster'),
queue.wrapRequestHandlerTask(state, handleStartCluster))
def subscribe(mq, state):
url = (state.conf('www.url_prefix') +
'/' +
os.path.basename(state.conf('clusters.addinstances_www')))
forwardOrCreate = forwardOrCreateTask(url,
state.conf('clusters.addinstances_queue'),
'addInstances',
4)
processAddPipe = defer_pipe.pipe([queue.keysInBody(['cluster',
'user_name',
'num_exec',
'num_data']),
forwardOrCreate])
processAddInstances = queue.returnResponse(processAddPipe)
queue.subscribe(mq,
state.conf('clusters.addinstances_www'),
state.conf('clusters.concurrent_addinstances'),
queue.wrapRequestHandler(state, processAddInstances))
queue.subscribe(mq,
state.conf('clusters.addinstances_queue'),
state.conf('clusters.concurrent_addinstances'),
queue.wrapRequestHandlerTask(state, handleAddInstances))
def subscribe(mq, state):
"""Subscribes to the queues needed to handle any incoming import cluster
requests.
"""
createAndForward = queue.createTaskAndForward(state.conf('clusters.importcluster_queue'),
'importCluster',
4)
processImportPipe = defer_pipe.pipe([queue.keysInBody(['host',
'cred_name',
'user_name',
'src_cluster',
'dst_cluster']),
returnClusterImportTaskIfExists,
createCluster,
createAndForward])
processImportCluster = queue.returnResponse(processImportPipe)
queue.subscribe(mq,
state.conf('clusters.importcluster_www'),
state.conf('clusters.concurrent_importcluster'),
queue.wrapRequestHandler(state, processImportCluster))
queue.subscribe(mq,
state.conf('clusters.importcluster_queue'),
state.conf('clusters.concurrent_importcluster'),
queue.wrapRequestHandlerTask(state, handleImportCluster))
def makeService(conf):
mqService = client.makeService(conf)
mqFactory = mqService.mqFactory
processRequest = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'tasklet',
'conf']),
forwardOrCreate(
conf('www.url_prefix') + '/' +
os.path.basename(conf('tasklets.tasklets_www')),
conf('tasklets.tasklets_queue'),
'runTasklets',
1)]))
queue.subscribe(mqFactory,
conf('tasklets.tasklets_www'),
conf('tasklets.concurrent_tasklets'),
queue.wrapRequestHandler(None, processRequest))
processRunTasklet = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['tasklet',
'conf']),
handleRunTasklet]),
queue.failureMsg)
queue.subscribe(mqFactory,
conf('tasklets.tasklets_queue'),
conf('tasklets.concurrent_tasklets'),
queue.wrapRequestHandlerTask(None, processRunTasklet))
return mqService
def subscribe(mq, state):
createAndForward = queue.createTaskAndForward(
state.conf('clusters.terminatecluster_queue'), 'terminateCluster', 1)
processTerminatePipe = defer_pipe.pipe(
[queue.keysInBody(['cluster_name', 'user_name']), createAndForward])
processTerminateCluster = queue.returnResponse(processTerminatePipe)
queue.subscribe(mq, state.conf('clusters.terminatecluster_www'),
state.conf('clusters.concurrent_terminatecluster'),
queue.wrapRequestHandler(state, processTerminateCluster))
queue.subscribe(
mq, state.conf('clusters.terminatecluster_queue'),
state.conf('clusters.concurrent_terminatecluster'),
queue.wrapRequestHandlerTask(state, handleTerminateCluster))
def subscribe(mq, state):
processTagData = queue.returnResponse(
defer_pipe.pipe([
queue.keysInBody(['cluster', 'tag_name', 'metadata', 'action']),
_forwardToCluster(state.conf, state.conf('tags.createupdate_www')),
_validateAction,
queue.createTaskAndForward(state.conf('tags.createupdate_queue'),
'tagCreateUpdate', 0), _returnTag
]))
queue.subscribe(mq, state.conf('tags.createupdate_www'),
state.conf('tags.concurrent_createupdate'),
queue.wrapRequestHandler(state, processTagData))
queue.subscribe(mq, state.conf('tags.createupdate_queue'),
state.conf('tags.concurrent_createupdate'),
queue.wrapRequestHandlerTask(state, handleTaskTagData))
def subscribe(mq, state):
processDeleteTag = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'tag_name']),
_forwardToCluster(state.conf,
state.conf('tags.delete_www')),
queue.createTaskAndForward(state.conf('tags.delete_queue'),
'deleteTag',
0)]))
queue.subscribe(mq,
state.conf('tags.delete_www'),
state.conf('tags.concurrent_delete'),
queue.wrapRequestHandler(state, processDeleteTag))
queue.subscribe(mq,
state.conf('tags.delete_queue'),
state.conf('tags.concurrent_delete'),
queue.wrapRequestHandlerTask(state, handleDeleteTag))
def subscribe(mq, state):
createAndForward = queue.createTaskAndForward(state.conf("clusters.terminatecluster_queue"), "terminateCluster", 1)
processTerminatePipe = defer_pipe.pipe([queue.keysInBody(["cluster_name", "user_name"]), createAndForward])
processTerminateCluster = queue.returnResponse(processTerminatePipe)
queue.subscribe(
mq,
state.conf("clusters.terminatecluster_www"),
state.conf("clusters.concurrent_terminatecluster"),
queue.wrapRequestHandler(state, processTerminateCluster),
)
queue.subscribe(
mq,
state.conf("clusters.terminatecluster_queue"),
state.conf("clusters.concurrent_terminatecluster"),
queue.wrapRequestHandlerTask(state, handleTerminateCluster),
)
def subscribe(mq, state):
processTransferTag = queue.returnResponse(
defer_pipe.pipe([
queue.keysInBody([
'cluster', 'user_name', 'tag_name', 'src_cluster',
'dst_cluster', 'dst_type'
]),
_forwardToCluster(state.conf, state.conf('tags.transfer_www')),
queue.createTaskAndForward(state.conf('tags.transfer_queue'),
'transferTag', 0)
]))
queue.subscribe(mq, state.conf('tags.transfer_www'),
state.conf('tags.concurrent_transfer'),
queue.wrapRequestHandler(state, processTransferTag))
queue.subscribe(mq, state.conf('tags.transfer_queue'),
state.conf('tags.concurrent_transfer'),
queue.wrapRequestHandlerTask(state, handleTransferTag))
def subscribe(mq, state):
url = (state.conf('www.url_prefix') + '/' +
os.path.basename(state.conf('clusters.addinstances_www')))
forwardOrCreate = forwardOrCreateTask(
url, state.conf('clusters.addinstances_queue'), 'addInstances', 4)
processAddPipe = defer_pipe.pipe([
queue.keysInBody(['cluster', 'user_name', 'num_exec', 'num_data']),
forwardOrCreate
])
processAddInstances = queue.returnResponse(processAddPipe)
queue.subscribe(mq, state.conf('clusters.addinstances_www'),
state.conf('clusters.concurrent_addinstances'),
queue.wrapRequestHandler(state, processAddInstances))
queue.subscribe(mq, state.conf('clusters.addinstances_queue'),
state.conf('clusters.concurrent_addinstances'),
queue.wrapRequestHandlerTask(state, handleAddInstances))
def subscribe(mq, state):
processRealizePhantom = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'tag_name',
'user_name',
'phantom',
'metadata']),
_forwardToCluster(state.conf,
state.conf('tags.realize_www')),
queue.createTaskAndForward(state.conf('tags.realize_queue'),
'realizePhantom',
0)]))
queue.subscribe(mq,
state.conf('tags.realize_www'),
state.conf('tags.concurrent_realize'),
queue.wrapRequestHandler(state, processRealizePhantom))
queue.subscribe(mq,
state.conf('tags.realize_queue'),
state.conf('tags.concurrent_realize'),
queue.wrapRequestHandlerTask(state, handleRealizePhantom))
def subscribe(mq, state):
createAndForward = queue.createTaskAndForward(
state.conf('clusters.startcluster_queue'), 'startCluster', 5)
processStartPipe = defer_pipe.pipe([
queue.keysInBody([
'cluster_name', 'user_name', 'num_exec', 'num_data', 'cred_name',
'conf'
]), returnClusterStartTaskIfExists, createCluster, createAndForward
])
processStartCluster = queue.returnResponse(processStartPipe)
queue.subscribe(mq, state.conf('clusters.startcluster_www'),
state.conf('clusters.concurrent_startcluster'),
queue.wrapRequestHandler(state, processStartCluster))
queue.subscribe(mq, state.conf('clusters.startcluster_queue'),
state.conf('clusters.concurrent_startcluster'),
queue.wrapRequestHandlerTask(state, handleStartCluster))
def subscribe(mq, state):
createAndForward = queue.createTaskAndForward(
state.conf('clusters.terminateinstances_queue'), 'terminateInstances',
1)
processTerminateInstancesPipe = defer_pipe.pipe([
queue.keysInBody(
['cluster_name', 'user_name', 'by_attribute', 'attribute_values']),
createAndForward
])
processTerminateInstances = queue.returnResponse(
processTerminateInstancesPipe)
queue.subscribe(mq, state.conf('clusters.terminateinstances_www'),
state.conf('clusters.concurrent_terminateinstances'),
queue.wrapRequestHandler(state, processTerminateInstances))
queue.subscribe(
mq, state.conf('clusters.terminateinstances_queue'),
state.conf('clusters.concurrent_terminateinstances'),
queue.wrapRequestHandlerTask(state, handleTerminateInstances))
def subscribe(mq, state):
createAndForward = queue.createTaskAndForward(state.conf('clusters.terminateinstances_queue'),
'terminateInstances',
1)
processTerminateInstancesPipe = defer_pipe.pipe([queue.keysInBody(['cluster_name',
'user_name',
'by_attribute',
'attribute_values']),
createAndForward])
processTerminateInstances = queue.returnResponse(processTerminateInstancesPipe)
queue.subscribe(mq,
state.conf('clusters.terminateinstances_www'),
state.conf('clusters.concurrent_terminateinstances'),
queue.wrapRequestHandler(state, processTerminateInstances))
queue.subscribe(mq,
state.conf('clusters.terminateinstances_queue'),
state.conf('clusters.concurrent_terminateinstances'),
queue.wrapRequestHandlerTask(state, handleTerminateInstances))
def subscribe(mq, state):
processTagData = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'tag_name',
'metadata',
'action']),
_forwardToCluster(state.conf,
state.conf('tags.createupdate_www')),
_validateAction,
queue.createTaskAndForward(state.conf('tags.createupdate_queue'),
'tagCreateUpdate',
0),
_returnTag]))
queue.subscribe(mq,
state.conf('tags.createupdate_www'),
state.conf('tags.concurrent_createupdate'),
queue.wrapRequestHandler(state, processTagData))
queue.subscribe(mq,
state.conf('tags.createupdate_queue'),
state.conf('tags.concurrent_createupdate'),
queue.wrapRequestHandlerTask(state, handleTaskTagData))
def subscribe(mq, state):
processTransferTag = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'user_name',
'tag_name',
'src_cluster',
'dst_cluster',
'dst_type']),
_forwardToCluster(state.conf,
state.conf('tags.transfer_www')),
queue.createTaskAndForward(state.conf('tags.transfer_queue'),
'transferTag',
0)]))
queue.subscribe(mq,
state.conf('tags.transfer_www'),
state.conf('tags.concurrent_transfer'),
queue.wrapRequestHandler(state, processTransferTag))
queue.subscribe(mq,
state.conf('tags.transfer_queue'),
state.conf('tags.concurrent_transfer'),
queue.wrapRequestHandlerTask(state, handleTransferTag))
def subscribe(mq, state):
"""Subscribes to the queues needed to handle any incoming import cluster
requests.
"""
createAndForward = queue.createTaskAndForward(
state.conf('clusters.importcluster_queue'), 'importCluster', 4)
processImportPipe = defer_pipe.pipe([
queue.keysInBody(
['host', 'cred_name', 'user_name', 'src_cluster', 'dst_cluster']),
returnClusterImportTaskIfExists, createCluster, createAndForward
])
processImportCluster = queue.returnResponse(processImportPipe)
queue.subscribe(mq, state.conf('clusters.importcluster_www'),
state.conf('clusters.concurrent_importcluster'),
queue.wrapRequestHandler(state, processImportCluster))
queue.subscribe(mq, state.conf('clusters.importcluster_queue'),
state.conf('clusters.concurrent_importcluster'),
queue.wrapRequestHandlerTask(state, handleImportCluster))
