def makeService(conf):
mqService = client.makeService(conf)
mqFactory = mqService.mqFactory
processRequest = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'tasklet',
'conf']),
forwardOrCreate(
conf('www.url_prefix') + '/' +
os.path.basename(conf('tasklets.tasklets_www')),
conf('tasklets.tasklets_queue'),
'runTasklets',
1)]))
queue.subscribe(mqFactory,
conf('tasklets.tasklets_www'),
conf('tasklets.concurrent_tasklets'),
queue.wrapRequestHandler(None, processRequest))
processRunTasklet = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['tasklet',
'conf']),
handleRunTasklet]),
queue.failureMsg)
queue.subscribe(mqFactory,
conf('tasklets.tasklets_queue'),
conf('tasklets.concurrent_tasklets'),
queue.wrapRequestHandlerTask(None, processRunTasklet))
return mqService
def makeService(conf):
mqService = client.makeService(conf)
mqFactory = mqService.mqFactory
processRequest = queue.returnResponse(
defer_pipe.pipe([
queue.keysInBody(['cluster', 'tasklet', 'conf']),
forwardOrCreate(
conf('www.url_prefix') + '/' +
os.path.basename(conf('tasklets.tasklets_www')),
conf('tasklets.tasklets_queue'), 'runTasklets', 1)
]))
queue.subscribe(mqFactory, conf('tasklets.tasklets_www'),
conf('tasklets.concurrent_tasklets'),
queue.wrapRequestHandler(None, processRequest))
processRunTasklet = defer_pipe.hookError(
defer_pipe.pipe(
[queue.keysInBody(['tasklet', 'conf']), handleRunTasklet]),
queue.failureMsg)
queue.subscribe(mqFactory, conf('tasklets.tasklets_queue'),
conf('tasklets.concurrent_tasklets'),
queue.wrapRequestHandlerTask(None, processRunTasklet))
return mqService
def wrapRequestHandler(state, f):
_addState = lambda r : defer_pipe.ret(r.update(state=state))
return defer_pipe.pipe([decodeRequest,
defer_pipe.hookError(defer_pipe.pipe([keysInBody(['return_queue']),
_addState,
f,
ackMsg]),
ackMsgFailure)])
def wrapRequestHandler(state, f):
_addState = lambda r: defer_pipe.ret(r.update(state=state))
return defer_pipe.pipe([
decodeRequest,
defer_pipe.hookError(
defer_pipe.pipe(
[keysInBody(['return_queue']), _addState, f, ackMsg]),
ackMsgFailure)
])
def subscribe(mq, state):
"""Subscribes to the queues needed to handle any incoming import cluster
requests.
"""
createAndForward = queue.createTaskAndForward(state.conf('clusters.importcluster_queue'),
'importCluster',
4)
processImportPipe = defer_pipe.pipe([queue.keysInBody(['host',
'cred_name',
'user_name',
'src_cluster',
'dst_cluster']),
returnClusterImportTaskIfExists,
createCluster,
createAndForward])
processImportCluster = queue.returnResponse(processImportPipe)
queue.subscribe(mq,
state.conf('clusters.importcluster_www'),
state.conf('clusters.concurrent_importcluster'),
queue.wrapRequestHandler(state, processImportCluster))
queue.subscribe(mq,
state.conf('clusters.importcluster_queue'),
state.conf('clusters.concurrent_importcluster'),
queue.wrapRequestHandlerTask(state, handleImportCluster))
def subscribe(mq, state):
url = (state.conf('www.url_prefix') +
'/' +
os.path.basename(state.conf('clusters.addinstances_www')))
forwardOrCreate = forwardOrCreateTask(url,
state.conf('clusters.addinstances_queue'),
'addInstances',
4)
processAddPipe = defer_pipe.pipe([queue.keysInBody(['cluster',
'user_name',
'num_exec',
'num_data']),
forwardOrCreate])
processAddInstances = queue.returnResponse(processAddPipe)
queue.subscribe(mq,
state.conf('clusters.addinstances_www'),
state.conf('clusters.concurrent_addinstances'),
queue.wrapRequestHandler(state, processAddInstances))
queue.subscribe(mq,
state.conf('clusters.addinstances_queue'),
state.conf('clusters.concurrent_addinstances'),
queue.wrapRequestHandlerTask(state, handleAddInstances))
def forwardOrCreate(url, dstQueue, tType, numTasks):
# Might not need the runPipeCurry after switching to queue.returnResponse
# but keeping it for now beause I don't think it does any harm
return defer_pipe.runPipeCurry(defer_pipe.pipe([queue.forwardRequestToCluster(url),
queue.createTaskAndForward(dstQueue,
tType,
numTasks)]))
def subscribe(mq, state):
processPipe = defer_pipe.pipe(
[queue.keysInBody(['user_name']), handleList])
processClusterList = queue.returnResponse(processPipe)
queue.subscribe(mq, state.conf('clusters.listclusters_www'),
state.conf('clusters.concurrent_listclusters'),
queue.wrapRequestHandler(state, processClusterList))
def subscribe(mq, state):
processPipe = defer_pipe.pipe(
[queue.keysInBody(['user_name', 'cluster_name']), handleConfig])
processClusterConfig = queue.returnResponse(processPipe)
queue.subscribe(mq, state.conf('clusters.config_www'),
state.conf('clusters.concurrent_config'),
queue.wrapRequestHandler(state, processClusterConfig))
def subscribe(mq, state):
createAndForward = queue.createTaskAndForward(state.conf('clusters.startcluster_queue'),
'startCluster',
5)
processStartPipe = defer_pipe.pipe([queue.keysInBody(['cluster_name',
'user_name',
'num_exec',
'num_data',
'cred_name',
'conf']),
returnClusterStartTaskIfExists,
createCluster,
createAndForward])
processStartCluster = queue.returnResponse(processStartPipe)
queue.subscribe(mq,
state.conf('clusters.startcluster_www'),
state.conf('clusters.concurrent_startcluster'),
queue.wrapRequestHandler(state, processStartCluster))
queue.subscribe(mq,
state.conf('clusters.startcluster_queue'),
state.conf('clusters.concurrent_startcluster'),
queue.wrapRequestHandlerTask(state, handleStartCluster))
def forwardOrCreate(url, dstQueue, tType, numTasks):
# Might not need the runPipeCurry after switching to queue.returnResponse
# but keeping it for now beause I don't think it does any harm
return defer_pipe.runPipeCurry(
defer_pipe.pipe([
queue.forwardRequestToCluster(url),
queue.createTaskAndForward(dstQueue, tType, numTasks)
]))
def subscribe(mq, state):
processPipe = defer_pipe.pipe([queue.keysInBody(['user_name']),
handleList])
processClusterList = queue.returnResponse(processPipe)
queue.subscribe(mq,
state.conf('clusters.listclusters_www'),
state.conf('clusters.concurrent_listclusters'),
queue.wrapRequestHandler(state, processClusterList))
def _subscribe(mq, state):
yield _getVMInfo(state)
yield _checkForUpdatesAndLoop(state)
processInfo = queue.returnResponse(defer_pipe.pipe([handleWWWInfo]))
queue.subscribe(mq, state.conf('vm.info_www'),
state.conf('vm.concurrent_info'),
queue.wrapRequestHandler(state, processInfo))
def subscribe(mq, state):
processGetCType = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name']),
credentials_misc.loadCredentialForRequest,
handleGetCType]),
queue.failureMsg)
queue.subscribe(mq,
state.conf('credentials.getctype_queue'),
state.conf('credentials.concurrent_getctype'),
queue.wrapRequestHandler(state, processGetCType))
def subscribe(mq, state):
processListKeypairs = defer_pipe.hookError(
defer_pipe.pipe([
queue.keysInBody(['credential_name']),
credentials_misc.loadCredentialForRequest, handleListKeypairs
]), queue.failureMsg)
queue.subscribe(mq, state.conf('credentials.listkeypairs_queue'),
state.conf('credentials.concurrent_listkeypairs'),
queue.wrapRequestHandler(state, processListKeypairs))
def subscribe(mq, state):
processRunPipeline = queue.returnResponse(
defer_pipe.pipe([
queue.keysInBody(['cluster', 'user_name', 'bare_run', 'config']),
pipeline_misc.containsPipelineTemplate, handleWWWRunPipeline
]))
queue.subscribe(mq, state.conf('pipelines.run_www'),
state.conf('pipelines.concurrent_run'),
queue.wrapRequestHandler(state, processRunPipeline))
def subscribe(mq, state):
processListKeypairs = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name']),
credentials_misc.loadCredentialForRequest,
handleListKeypairs]),
queue.failureMsg)
queue.subscribe(mq,
state.conf('credentials.listkeypairs_queue'),
state.conf('credentials.concurrent_listkeypairs'),
queue.wrapRequestHandler(state, processListKeypairs))
def subscribe(mq, state):
processUpdateInstances = defer_pipe.hookError(
defer_pipe.pipe([
queue.keysInBody(['credential_name', 'instances']),
credentials_misc.loadCredentialForRequest, handleUpdateInstances
]), queue.failureMsg)
queue.subscribe(mq, state.conf('credentials.updateinstances_queue'),
state.conf('credentials.concurrent_updateinstances'),
queue.wrapRequestHandler(state, processUpdateInstances))
def subscribe(mq, state):
processGetCType = defer_pipe.hookError(
defer_pipe.pipe([
queue.keysInBody(['credential_name']),
credentials_misc.loadCredentialForRequest, handleGetCType
]), queue.failureMsg)
queue.subscribe(mq, state.conf('credentials.getctype_queue'),
state.conf('credentials.concurrent_getctype'),
queue.wrapRequestHandler(state, processGetCType))
def subscribe(mq, state):
processCredentialConfig = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name']),
credentials_misc.loadCredentialForRequest,
handleCredentialConfig]),
queue.failureMsg)
queue.subscribe(mq,
state.conf('credentials.credentialconfig_queue'),
state.conf('credentials.concurrent_credentialconfig'),
queue.wrapRequestHandler(state, processCredentialConfig))
def subscribe(mq, state):
processCredentialConfig = defer_pipe.hookError(
defer_pipe.pipe([
queue.keysInBody(['credential_name']),
credentials_misc.loadCredentialForRequest, handleCredentialConfig
]), queue.failureMsg)
queue.subscribe(mq, state.conf('credentials.credentialconfig_queue'),
state.conf('credentials.concurrent_credentialconfig'),
queue.wrapRequestHandler(state, processCredentialConfig))
def subscribe(mq, state):
processPipe = defer_pipe.pipe([queue.keysInBody(['user_name',
'cluster_name']),
handleConfig])
processClusterConfig = queue.returnResponse(processPipe)
queue.subscribe(mq,
state.conf('clusters.config_www'),
state.conf('clusters.concurrent_config'),
queue.wrapRequestHandler(state, processClusterConfig))
def _subscribe(self):
processEvent = defer_pipe.pipe([
queue.keysInBody([
'id', 'file', 'event', 'retval', 'props', 'host', 'time',
'name', 'message'
]), self._handleEventMessage
])
queue.subscribe(self.mq, self._queueName(), 1,
queue.wrapRequestHandler(None, processEvent))
def subscribe(mq, state):
processAddKeypair = defer_pipe.hookError(
defer_pipe.pipe([
queue.keysInBody(['credential_name', 'keypair_name']),
credentials_misc.loadCredentialForRequest, handleAddKeypair
]), queue.failureMsg)
queue.subscribe(mq, state.conf('credentials.addkeypair_queue'),
state.conf('credentials.concurrent_addkeypair'),
queue.wrapRequestHandler(state, processAddKeypair))
def subscribe(mq, state):
processTerminateInstances = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name',
'instances']),
credentials_misc.loadCredentialForRequest,
handleTerminateInstances]),
queue.failureMsg)
queue.subscribe(mq,
state.conf('credentials.terminateinstances_queue'),
state.conf('credentials.concurrent_terminateinstances'),
queue.wrapRequestHandler(state, processTerminateInstances))
def subscribe(mq, state):
processListProtocols = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'user_name']),
pipeline_misc.forwardToCluster(state.conf,
state.conf('pipelines.listprotocols_www')),
handleWWWListProtocols]))
queue.subscribe(mq,
state.conf('pipelines.listprotocols_www'),
state.conf('pipelines.concurrent_listprotocols'),
queue.wrapRequestHandler(state, processListProtocols))
def subscribe(mq, state):
processPipelineDelete = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'user_name',
'criteria']),
_forwardToCluster(state.conf, state.conf('pipelines.delete_www')),
handleWWWPipelineDelete]))
queue.subscribe(mq,
state.conf('pipelines.delete_www'),
state.conf('pipelines.concurrent_delete'),
queue.wrapRequestHandler(state, processPipelineDelete))
def subscribe(mq, state):
processAddGroup = defer_pipe.hookError(
defer_pipe.pipe([
queue.keysInBody(
['credential_name', 'group_name', 'group_description']),
credentials_misc.loadCredentialForRequest, handleAddGroup
]), queue.failureMsg)
queue.subscribe(mq, state.conf('credentials.addgroup_queue'),
state.conf('credentials.concurrent_addgroup'),
queue.wrapRequestHandler(state, processAddGroup))
def subscribe(mq, state):
processPipelineDelete = queue.returnResponse(
defer_pipe.pipe([
queue.keysInBody(['cluster', 'user_name', 'criteria']),
_forwardToCluster(state.conf, state.conf('pipelines.delete_www')),
handleWWWPipelineDelete
]))
queue.subscribe(mq, state.conf('pipelines.delete_www'),
state.conf('pipelines.concurrent_delete'),
queue.wrapRequestHandler(state, processPipelineDelete))
def subscribe(mq, state):
processObserver = queue.returnResponse(
defer_pipe.pipe([
queue.keysInBody([
'id', 'file', 'event', 'retval', 'props', 'host', 'time',
'name', 'message'
]), handleWWWObserver
]))
queue.subscribe(mq, state.conf('pipelines.observer_www'),
state.conf('pipelines.concurrent_observer'),
queue.wrapRequestHandler(state, processObserver))
def subscribe(mq, state):
processTaskList = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'user_name']),
_forwardToCluster(state.conf,
state.conf('tasks.list_www')),
handleTaskList]))
queue.subscribe(mq,
state.conf('tasks.list_www'),
state.conf('tasks.concurrent_list'),
queue.wrapRequestHandler(state, processTaskList))
def subscribe(mq, state):
processAuthorizeGroup = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name',
'group_name',
'port_range']),
credentials_misc.loadCredentialForRequest,
handleAuthorizeGroup]),
queue.failureMsg)
queue.subscribe(mq,
state.conf('credentials.authorizegroup_queue'),
state.conf('credentials.concurrent_authorizegroup'),
queue.wrapRequestHandler(state, processAuthorizeGroup))
def subscribe(mq, state):
processResumePipeline = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'user_name',
'pipeline_name']),
pipeline_misc.forwardToCluster(state.conf,
state.conf('pipelines.resume_www')),
handleWWWResumePipeline]))
queue.subscribe(mq,
state.conf('pipelines.resume_www'),
state.conf('pipelines.concurrent_resume'),
queue.wrapRequestHandler(state, processResumePipeline))
def subscribe(mq, state):
processRunInstances = defer_pipe.hookError(
defer_pipe.pipe([
queue.keysInBody([
'credential_name', 'ami', 'key', 'instance_type', 'groups',
'num_instances'
]), credentials_misc.loadCredentialForRequest, handleRunInstances
]), queue.failureMsg)
queue.subscribe(mq, state.conf('credentials.runinstances_queue'),
state.conf('credentials.concurrent_runinstances'),
queue.wrapRequestHandler(state, processRunInstances))
def subscribe(mq, state):
processWWWDeleteCredential = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name']),
queue.forwardRequestToCluster(
state.conf('www.url_prefix') + '/' +
os.path.basename(state.conf('credentials.delete_www'))),
handleWWWDeleteCredential]),
queue.failureMsg)
queue.subscribe(mq,
state.conf('credentials.delete_www'),
state.conf('credentials.concurrent_deletecredential'),
queue.wrapRequestHandler(state, processWWWDeleteCredential))
def subscribe(mq, state):
processWWWListAddCredentials = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['cluster']),
queue.forwardRequestToCluster(
state.conf('www.url_prefix') + '/' +
os.path.basename(state.conf('credentials.listaddcredentials_www'))),
handleWWWListAddCredentials]),
queue.failureMsg)
queue.subscribe(mq,
state.conf('credentials.listaddcredentials_www'),
state.conf('credentials.concurrent_listaddcredentials'),
queue.wrapRequestHandler(state, processWWWListAddCredentials))
def subscribe(mq, state):
processListProtocols = queue.returnResponse(
defer_pipe.pipe([
queue.keysInBody(['cluster', 'user_name']),
pipeline_misc.forwardToCluster(
state.conf, state.conf('pipelines.listprotocols_www')),
handleWWWListProtocols
]))
queue.subscribe(mq, state.conf('pipelines.listprotocols_www'),
state.conf('pipelines.concurrent_listprotocols'),
queue.wrapRequestHandler(state, processListProtocols))
def subscribe(mq, state):
processRunPipeline = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'user_name',
'bare_run',
'config']),
pipeline_misc.containsPipelineTemplate,
handleWWWRunPipeline]))
queue.subscribe(mq,
state.conf('pipelines.run_www'),
state.conf('pipelines.concurrent_run'),
queue.wrapRequestHandler(state, processRunPipeline))
def subscribe(mq, state):
processAddKeypair = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name',
'keypair_name']),
credentials_misc.loadCredentialForRequest,
handleAddKeypair]),
queue.failureMsg)
queue.subscribe(mq,
state.conf('credentials.addkeypair_queue'),
state.conf('credentials.concurrent_addkeypair'),
queue.wrapRequestHandler(state, processAddKeypair))
def subscribe(mq, state):
processResumePipeline = queue.returnResponse(
defer_pipe.pipe([
queue.keysInBody(['cluster', 'user_name', 'pipeline_name']),
pipeline_misc.forwardToCluster(state.conf,
state.conf('pipelines.resume_www')),
handleWWWResumePipeline
]))
queue.subscribe(mq, state.conf('pipelines.resume_www'),
state.conf('pipelines.concurrent_resume'),
queue.wrapRequestHandler(state, processResumePipeline))
def subscribe(mq, state):
processTaskList = queue.returnResponse(
defer_pipe.pipe([
queue.keysInBody(['cluster', 'user_name']),
_forwardToCluster(state.conf, state.conf('tasks.list_www')),
handleTaskList
]))
queue.subscribe(mq, state.conf('tasks.list_www'),
state.conf('tasks.concurrent_list'),
queue.wrapRequestHandler(state, processTaskList))
def subscribe(mq, state):
processValidatePipelineConfig = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'bare_run',
'config']),
pipeline_misc.containsPipelineTemplate,
pipeline_misc.forwardToCluster(state.conf,
state.conf('pipelines.validate_www')),
handleWWWValidatePipelineConfig]))
queue.subscribe(mq,
state.conf('pipelines.validate_www'),
state.conf('pipelines.concurrent_validate'),
queue.wrapRequestHandler(state, processValidatePipelineConfig))
def subscribe(mq, state):
processUpdatePipelineConfig = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'user_name',
'criteria',
'config']),
pipeline_misc.forwardToCluster(state.conf,
state.conf('pipelines.update_www')),
handleWWWUpdatePipelineConfig]))
queue.subscribe(mq,
state.conf('pipelines.update_www'),
state.conf('pipelines.concurrent_update'),
queue.wrapRequestHandler(state, processUpdatePipelineConfig))
def subscribe(mq, state):
processUpdatePipelineConfig = queue.returnResponse(
defer_pipe.pipe([
queue.keysInBody(['cluster', 'user_name', 'criteria', 'config']),
pipeline_misc.forwardToCluster(state.conf,
state.conf('pipelines.update_www')),
handleWWWUpdatePipelineConfig
]))
queue.subscribe(
mq, state.conf('pipelines.update_www'),
state.conf('pipelines.concurrent_update'),
queue.wrapRequestHandler(state, processUpdatePipelineConfig))
def subscribe(mq, state):
yield defer_utils.tryUntil(10,
lambda : _monitorAnyPipelines(mq, state),
onFailure=defer_utils.sleep(2))
processPipelineList = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'user_name']),
_forwardToCluster(state.conf, state.conf('pipelines.list_www')),
handleWWWPipelineList]))
queue.subscribe(mq,
state.conf('pipelines.list_www'),
state.conf('pipelines.concurrent_list'),
queue.wrapRequestHandler(state, processPipelineList))
def subscribe(mq, state):
processWWWListAddCredentials = defer_pipe.hookError(
defer_pipe.pipe([
queue.keysInBody(['cluster']),
queue.forwardRequestToCluster(
state.conf('www.url_prefix') + '/' + os.path.basename(
state.conf('credentials.listaddcredentials_www'))),
handleWWWListAddCredentials
]), queue.failureMsg)
queue.subscribe(
mq, state.conf('credentials.listaddcredentials_www'),
state.conf('credentials.concurrent_listaddcredentials'),
queue.wrapRequestHandler(state, processWWWListAddCredentials))
def subscribe(mq, state):
processValidatePipelineConfig = queue.returnResponse(
defer_pipe.pipe([
queue.keysInBody(['cluster', 'bare_run', 'config']),
pipeline_misc.containsPipelineTemplate,
pipeline_misc.forwardToCluster(
state.conf, state.conf('pipelines.validate_www')),
handleWWWValidatePipelineConfig
]))
queue.subscribe(
mq, state.conf('pipelines.validate_www'),
state.conf('pipelines.concurrent_validate'),
queue.wrapRequestHandler(state, processValidatePipelineConfig))
def subscribe(mq, state):
processPipelineCreate = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'user_name',
'pipeline_name',
'protocol',
'queue',
'config']),
pipeline_misc.forwardToCluster(state.conf,
state.conf('pipelines.create_www')),
handleWWWPipelineCreate]))
queue.subscribe(mq,
state.conf('pipelines.create_www'),
state.conf('pipelines.concurrent_create'),
queue.wrapRequestHandler(state, processPipelineCreate))
def subscribe(mq, state):
processRunInstances = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name',
'ami',
'key',
'instance_type',
'groups',
'num_instances']),
credentials_misc.loadCredentialForRequest,
handleRunInstances]),
queue.failureMsg)
queue.subscribe(mq,
state.conf('credentials.runinstances_queue'),
state.conf('credentials.concurrent_runinstances'),
queue.wrapRequestHandler(state, processRunInstances))
def subscribe(mq, state):
yield defer_utils.tryUntil(10,
lambda: _monitorAnyPipelines(mq, state),
onFailure=defer_utils.sleep(2))
processPipelineList = queue.returnResponse(
defer_pipe.pipe([
queue.keysInBody(['cluster', 'user_name']),
_forwardToCluster(state.conf, state.conf('pipelines.list_www')),
handleWWWPipelineList
]))
queue.subscribe(mq, state.conf('pipelines.list_www'),
state.conf('pipelines.concurrent_list'),
queue.wrapRequestHandler(state, processPipelineList))
def subscribe(mq, state):
processObserver = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['id',
'file',
'event',
'retval',
'props',
'host',
'time',
'name',
'message']),
handleWWWObserver]))
queue.subscribe(mq,
state.conf('pipelines.observer_www'),
state.conf('pipelines.concurrent_observer'),
queue.wrapRequestHandler(state, processObserver))
def subscribe(mq, state):
createAndForward = queue.createTaskAndForward(
state.conf('clusters.terminatecluster_queue'), 'terminateCluster', 1)
processTerminatePipe = defer_pipe.pipe(
[queue.keysInBody(['cluster_name', 'user_name']), createAndForward])
processTerminateCluster = queue.returnResponse(processTerminatePipe)
queue.subscribe(mq, state.conf('clusters.terminatecluster_www'),
state.conf('clusters.concurrent_terminatecluster'),
queue.wrapRequestHandler(state, processTerminateCluster))
queue.subscribe(
mq, state.conf('clusters.terminatecluster_queue'),
state.conf('clusters.concurrent_terminatecluster'),
queue.wrapRequestHandlerTask(state, handleTerminateCluster))
def _subscribe(self):
processEvent = defer_pipe.pipe([queue.keysInBody(['id',
'file',
'event',
'retval',
'props',
'host',
'time',
'name',
'message']),
self._handleEventMessage])
queue.subscribe(self.mq,
self._queueName(),
1,
queue.wrapRequestHandler(None, processEvent))
def subscribe(mq, state):
processTagData = queue.returnResponse(
defer_pipe.pipe([
queue.keysInBody(['cluster', 'tag_name', 'metadata', 'action']),
_forwardToCluster(state.conf, state.conf('tags.createupdate_www')),
_validateAction,
queue.createTaskAndForward(state.conf('tags.createupdate_queue'),
'tagCreateUpdate', 0), _returnTag
]))
queue.subscribe(mq, state.conf('tags.createupdate_www'),
state.conf('tags.concurrent_createupdate'),
queue.wrapRequestHandler(state, processTagData))
queue.subscribe(mq, state.conf('tags.createupdate_queue'),
state.conf('tags.concurrent_createupdate'),
queue.wrapRequestHandlerTask(state, handleTaskTagData))
def subscribe(mq, state):
processUpdateInstances = defer_pipe.hookError(
defer_pipe.pipe(
[
queue.keysInBody(["credential_name", "instances"]),
credentials_misc.loadCredentialForRequest,
handleUpdateInstances,
]
),
queue.failureMsg,
)
queue.subscribe(
mq,
state.conf("credentials.updateinstances_queue"),
state.conf("credentials.concurrent_updateinstances"),
queue.wrapRequestHandler(state, processUpdateInstances),
)
def subscribe(mq, state):
processDeleteTag = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
'tag_name']),
_forwardToCluster(state.conf,
state.conf('tags.delete_www')),
queue.createTaskAndForward(state.conf('tags.delete_queue'),
'deleteTag',
0)]))
queue.subscribe(mq,
state.conf('tags.delete_www'),
state.conf('tags.concurrent_delete'),
queue.wrapRequestHandler(state, processDeleteTag))
queue.subscribe(mq,
state.conf('tags.delete_queue'),
state.conf('tags.concurrent_delete'),
queue.wrapRequestHandlerTask(state, handleDeleteTag))
def subscribe(mq, state):
processTransferTag = queue.returnResponse(
defer_pipe.pipe([
queue.keysInBody([
'cluster', 'user_name', 'tag_name', 'src_cluster',
'dst_cluster', 'dst_type'
]),
_forwardToCluster(state.conf, state.conf('tags.transfer_www')),
queue.createTaskAndForward(state.conf('tags.transfer_queue'),
'transferTag', 0)
]))
queue.subscribe(mq, state.conf('tags.transfer_www'),
state.conf('tags.concurrent_transfer'),
queue.wrapRequestHandler(state, processTransferTag))
queue.subscribe(mq, state.conf('tags.transfer_queue'),
state.conf('tags.concurrent_transfer'),
queue.wrapRequestHandlerTask(state, handleTransferTag))
def subscribe(mq, state):
createAndForward = queue.createTaskAndForward(
state.conf('clusters.startcluster_queue'), 'startCluster', 5)
processStartPipe = defer_pipe.pipe([
queue.keysInBody([
'cluster_name', 'user_name', 'num_exec', 'num_data', 'cred_name',
'conf'
]), returnClusterStartTaskIfExists, createCluster, createAndForward
])
processStartCluster = queue.returnResponse(processStartPipe)
queue.subscribe(mq, state.conf('clusters.startcluster_www'),
state.conf('clusters.concurrent_startcluster'),
queue.wrapRequestHandler(state, processStartCluster))
queue.subscribe(mq, state.conf('clusters.startcluster_queue'),
state.conf('clusters.concurrent_startcluster'),
queue.wrapRequestHandlerTask(state, handleStartCluster))