Esempio n. 1
0
def makeService(conf):
    mqService = client.makeService(conf)
    mqFactory = mqService.mqFactory


    processRequest = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                             'tasklet',
                                                                             'conf']),
                                                           forwardOrCreate(
                                                               conf('www.url_prefix') + '/' +
                                                               os.path.basename(conf('tasklets.tasklets_www')),
                                                               conf('tasklets.tasklets_queue'),
                                                               'runTasklets',
                                                               1)]))



    queue.subscribe(mqFactory,
                    conf('tasklets.tasklets_www'),
                    conf('tasklets.concurrent_tasklets'),
                    queue.wrapRequestHandler(None, processRequest))


    processRunTasklet = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['tasklet',
                                                                                'conf']),
                                                              handleRunTasklet]),
                                             queue.failureMsg)
    
    queue.subscribe(mqFactory,
                    conf('tasklets.tasklets_queue'),
                    conf('tasklets.concurrent_tasklets'),
                    queue.wrapRequestHandlerTask(None, processRunTasklet))
    
    
    return mqService
Esempio n. 2
0
def makeService(conf):
    mqService = client.makeService(conf)
    mqFactory = mqService.mqFactory

    processRequest = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'tasklet', 'conf']),
            forwardOrCreate(
                conf('www.url_prefix') + '/' +
                os.path.basename(conf('tasklets.tasklets_www')),
                conf('tasklets.tasklets_queue'), 'runTasklets', 1)
        ]))

    queue.subscribe(mqFactory, conf('tasklets.tasklets_www'),
                    conf('tasklets.concurrent_tasklets'),
                    queue.wrapRequestHandler(None, processRequest))

    processRunTasklet = defer_pipe.hookError(
        defer_pipe.pipe(
            [queue.keysInBody(['tasklet', 'conf']), handleRunTasklet]),
        queue.failureMsg)

    queue.subscribe(mqFactory, conf('tasklets.tasklets_queue'),
                    conf('tasklets.concurrent_tasklets'),
                    queue.wrapRequestHandlerTask(None, processRunTasklet))

    return mqService
Esempio n. 3
0
def wrapRequestHandler(state, f):
    _addState = lambda r : defer_pipe.ret(r.update(state=state))
    
    return defer_pipe.pipe([decodeRequest,
                            defer_pipe.hookError(defer_pipe.pipe([keysInBody(['return_queue']),
                                                                  _addState,
                                                                  f,
                                                                  ackMsg]),
                                                 ackMsgFailure)])
Esempio n. 4
0
def wrapRequestHandler(state, f):
    _addState = lambda r: defer_pipe.ret(r.update(state=state))

    return defer_pipe.pipe([
        decodeRequest,
        defer_pipe.hookError(
            defer_pipe.pipe(
                [keysInBody(['return_queue']), _addState, f, ackMsg]),
            ackMsgFailure)
    ])
Esempio n. 5
0
def subscribe(mq, state):
    """Subscribes to the queues needed to handle any incoming import cluster 
    requests.

    """
    createAndForward = queue.createTaskAndForward(state.conf('clusters.importcluster_queue'),
                                                  'importCluster',
                                                  4)

    processImportPipe = defer_pipe.pipe([queue.keysInBody(['host',
                                                           'cred_name',
                                                           'user_name',
                                                           'src_cluster',
                                                           'dst_cluster']),
                                        returnClusterImportTaskIfExists,
                                        createCluster,
                                        createAndForward])

    processImportCluster = queue.returnResponse(processImportPipe)

    queue.subscribe(mq,
                    state.conf('clusters.importcluster_www'),
                    state.conf('clusters.concurrent_importcluster'),
                    queue.wrapRequestHandler(state, processImportCluster))

    queue.subscribe(mq,
                    state.conf('clusters.importcluster_queue'),
                    state.conf('clusters.concurrent_importcluster'),
                    queue.wrapRequestHandlerTask(state, handleImportCluster))
Esempio n. 6
0
def subscribe(mq, state):
    url = (state.conf('www.url_prefix') +
           '/' +
           os.path.basename(state.conf('clusters.addinstances_www')))
    forwardOrCreate = forwardOrCreateTask(url,
                                          state.conf('clusters.addinstances_queue'),
                                          'addInstances',
                                          4)

    processAddPipe = defer_pipe.pipe([queue.keysInBody(['cluster',
                                                        'user_name',
                                                        'num_exec',
                                                        'num_data']),
                                      forwardOrCreate])
    
    processAddInstances = queue.returnResponse(processAddPipe)

    queue.subscribe(mq,
                    state.conf('clusters.addinstances_www'),
                    state.conf('clusters.concurrent_addinstances'),
                    queue.wrapRequestHandler(state, processAddInstances))

    queue.subscribe(mq,
                    state.conf('clusters.addinstances_queue'),
                    state.conf('clusters.concurrent_addinstances'),
                    queue.wrapRequestHandlerTask(state, handleAddInstances))
Esempio n. 7
0
def forwardOrCreate(url, dstQueue, tType, numTasks):
    # Might not need the runPipeCurry after switching to queue.returnResponse
    # but keeping it for now beause I don't think it does any harm
    return defer_pipe.runPipeCurry(defer_pipe.pipe([queue.forwardRequestToCluster(url),
                                                    queue.createTaskAndForward(dstQueue,
                                                                               tType,
                                                                               numTasks)]))
Esempio n. 8
0
def subscribe(mq, state):
    processPipe = defer_pipe.pipe(
        [queue.keysInBody(['user_name']), handleList])
    processClusterList = queue.returnResponse(processPipe)
    queue.subscribe(mq, state.conf('clusters.listclusters_www'),
                    state.conf('clusters.concurrent_listclusters'),
                    queue.wrapRequestHandler(state, processClusterList))
Esempio n. 9
0
def subscribe(mq, state):
    processPipe = defer_pipe.pipe(
        [queue.keysInBody(['user_name', 'cluster_name']), handleConfig])
    processClusterConfig = queue.returnResponse(processPipe)
    queue.subscribe(mq, state.conf('clusters.config_www'),
                    state.conf('clusters.concurrent_config'),
                    queue.wrapRequestHandler(state, processClusterConfig))
Esempio n. 10
0
def subscribe(mq, state):
    createAndForward = queue.createTaskAndForward(state.conf('clusters.startcluster_queue'),
                                                  'startCluster',
                                                  5)

    processStartPipe = defer_pipe.pipe([queue.keysInBody(['cluster_name',
                                                          'user_name',
                                                          'num_exec',
                                                          'num_data',
                                                          'cred_name',
                                                          'conf']),
                                        returnClusterStartTaskIfExists,
                                        createCluster,
                                        createAndForward])
    
    processStartCluster = queue.returnResponse(processStartPipe)

    queue.subscribe(mq,
                    state.conf('clusters.startcluster_www'),
                    state.conf('clusters.concurrent_startcluster'),
                    queue.wrapRequestHandler(state, processStartCluster))

    queue.subscribe(mq,
                    state.conf('clusters.startcluster_queue'),
                    state.conf('clusters.concurrent_startcluster'),
                    queue.wrapRequestHandlerTask(state, handleStartCluster))
Esempio n. 11
0
def forwardOrCreate(url, dstQueue, tType, numTasks):
    # Might not need the runPipeCurry after switching to queue.returnResponse
    # but keeping it for now beause I don't think it does any harm
    return defer_pipe.runPipeCurry(
        defer_pipe.pipe([
            queue.forwardRequestToCluster(url),
            queue.createTaskAndForward(dstQueue, tType, numTasks)
        ]))
Esempio n. 12
0
def subscribe(mq, state):
    processPipe = defer_pipe.pipe([queue.keysInBody(['user_name']),
                                   handleList])
    processClusterList = queue.returnResponse(processPipe)
    queue.subscribe(mq,
                    state.conf('clusters.listclusters_www'),
                    state.conf('clusters.concurrent_listclusters'),
                    queue.wrapRequestHandler(state, processClusterList))
Esempio n. 13
0
def _subscribe(mq, state):
    yield _getVMInfo(state)
    yield _checkForUpdatesAndLoop(state)

    processInfo = queue.returnResponse(defer_pipe.pipe([handleWWWInfo]))
    queue.subscribe(mq, state.conf('vm.info_www'),
                    state.conf('vm.concurrent_info'),
                    queue.wrapRequestHandler(state, processInfo))
Esempio n. 14
0
def subscribe(mq, state):
    processGetCType = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name']),
                                                            credentials_misc.loadCredentialForRequest,
                                                            handleGetCType]),
                                           queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.getctype_queue'),
                    state.conf('credentials.concurrent_getctype'),
                    queue.wrapRequestHandler(state, processGetCType))
Esempio n. 15
0
def subscribe(mq, state):
    processListKeypairs = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody(['credential_name']),
            credentials_misc.loadCredentialForRequest, handleListKeypairs
        ]), queue.failureMsg)
    queue.subscribe(mq, state.conf('credentials.listkeypairs_queue'),
                    state.conf('credentials.concurrent_listkeypairs'),
                    queue.wrapRequestHandler(state, processListKeypairs))
Esempio n. 16
0
def subscribe(mq, state):
    processRunPipeline = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'user_name', 'bare_run', 'config']),
            pipeline_misc.containsPipelineTemplate, handleWWWRunPipeline
        ]))
    queue.subscribe(mq, state.conf('pipelines.run_www'),
                    state.conf('pipelines.concurrent_run'),
                    queue.wrapRequestHandler(state, processRunPipeline))
Esempio n. 17
0
def subscribe(mq, state):
    processListKeypairs = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name']),
                                                                credentials_misc.loadCredentialForRequest,
                                                                handleListKeypairs]),
                                               queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.listkeypairs_queue'),
                    state.conf('credentials.concurrent_listkeypairs'),
                    queue.wrapRequestHandler(state, processListKeypairs))
def subscribe(mq, state):
    processUpdateInstances = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody(['credential_name', 'instances']),
            credentials_misc.loadCredentialForRequest, handleUpdateInstances
        ]), queue.failureMsg)
    queue.subscribe(mq, state.conf('credentials.updateinstances_queue'),
                    state.conf('credentials.concurrent_updateinstances'),
                    queue.wrapRequestHandler(state, processUpdateInstances))
Esempio n. 19
0
def subscribe(mq, state):
    processGetCType = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody(['credential_name']),
            credentials_misc.loadCredentialForRequest, handleGetCType
        ]), queue.failureMsg)
    queue.subscribe(mq, state.conf('credentials.getctype_queue'),
                    state.conf('credentials.concurrent_getctype'),
                    queue.wrapRequestHandler(state, processGetCType))
Esempio n. 20
0
def subscribe(mq, state):
    processCredentialConfig = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name']),
                                                                    credentials_misc.loadCredentialForRequest,
                                                                    handleCredentialConfig]),
                                                   queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.credentialconfig_queue'),
                    state.conf('credentials.concurrent_credentialconfig'),
                    queue.wrapRequestHandler(state, processCredentialConfig))
Esempio n. 21
0
def subscribe(mq, state):
    processCredentialConfig = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody(['credential_name']),
            credentials_misc.loadCredentialForRequest, handleCredentialConfig
        ]), queue.failureMsg)
    queue.subscribe(mq, state.conf('credentials.credentialconfig_queue'),
                    state.conf('credentials.concurrent_credentialconfig'),
                    queue.wrapRequestHandler(state, processCredentialConfig))
Esempio n. 22
0
def subscribe(mq, state):
    processPipe = defer_pipe.pipe([queue.keysInBody(['user_name',
                                                     'cluster_name']),
                                   handleConfig])
    processClusterConfig = queue.returnResponse(processPipe)
    queue.subscribe(mq,
                    state.conf('clusters.config_www'),
                    state.conf('clusters.concurrent_config'),
                    queue.wrapRequestHandler(state, processClusterConfig))
Esempio n. 23
0
    def _subscribe(self):
        processEvent = defer_pipe.pipe([
            queue.keysInBody([
                'id', 'file', 'event', 'retval', 'props', 'host', 'time',
                'name', 'message'
            ]), self._handleEventMessage
        ])

        queue.subscribe(self.mq, self._queueName(), 1,
                        queue.wrapRequestHandler(None, processEvent))
Esempio n. 24
0
def subscribe(mq, state):
    processAddKeypair = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody(['credential_name', 'keypair_name']),
            credentials_misc.loadCredentialForRequest, handleAddKeypair
        ]), queue.failureMsg)

    queue.subscribe(mq, state.conf('credentials.addkeypair_queue'),
                    state.conf('credentials.concurrent_addkeypair'),
                    queue.wrapRequestHandler(state, processAddKeypair))
def subscribe(mq, state):
    processTerminateInstances = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name',
                                                                                        'instances']),
                                                                      credentials_misc.loadCredentialForRequest,
                                                                      handleTerminateInstances]),
                                                     queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.terminateinstances_queue'),
                    state.conf('credentials.concurrent_terminateinstances'),
                    queue.wrapRequestHandler(state, processTerminateInstances))
Esempio n. 26
0
def subscribe(mq, state):
    processListProtocols = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                   'user_name']),
                                                                 pipeline_misc.forwardToCluster(state.conf,
                                                                                                state.conf('pipelines.listprotocols_www')),
                                                                 handleWWWListProtocols]))
    queue.subscribe(mq,
                    state.conf('pipelines.listprotocols_www'),
                    state.conf('pipelines.concurrent_listprotocols'),
                    queue.wrapRequestHandler(state, processListProtocols))
Esempio n. 27
0
def subscribe(mq, state):
    processPipelineDelete = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                    'user_name',
                                                                                    'criteria']),
                                                                  _forwardToCluster(state.conf, state.conf('pipelines.delete_www')),
                                                                  handleWWWPipelineDelete]))
    queue.subscribe(mq,
                    state.conf('pipelines.delete_www'),
                    state.conf('pipelines.concurrent_delete'),
                    queue.wrapRequestHandler(state, processPipelineDelete))
Esempio n. 28
0
def subscribe(mq, state):
    processAddGroup = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody(
                ['credential_name', 'group_name', 'group_description']),
            credentials_misc.loadCredentialForRequest, handleAddGroup
        ]), queue.failureMsg)
    queue.subscribe(mq, state.conf('credentials.addgroup_queue'),
                    state.conf('credentials.concurrent_addgroup'),
                    queue.wrapRequestHandler(state, processAddGroup))
Esempio n. 29
0
def subscribe(mq, state):
    processPipelineDelete = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'user_name', 'criteria']),
            _forwardToCluster(state.conf, state.conf('pipelines.delete_www')),
            handleWWWPipelineDelete
        ]))
    queue.subscribe(mq, state.conf('pipelines.delete_www'),
                    state.conf('pipelines.concurrent_delete'),
                    queue.wrapRequestHandler(state, processPipelineDelete))
Esempio n. 30
0
def subscribe(mq, state):
    processObserver = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody([
                'id', 'file', 'event', 'retval', 'props', 'host', 'time',
                'name', 'message'
            ]), handleWWWObserver
        ]))
    queue.subscribe(mq, state.conf('pipelines.observer_www'),
                    state.conf('pipelines.concurrent_observer'),
                    queue.wrapRequestHandler(state, processObserver))
Esempio n. 31
0
def subscribe(mq, state):
    processTaskList = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                              'user_name']),
                                                            _forwardToCluster(state.conf,
                                                                              state.conf('tasks.list_www')),
                                                            handleTaskList]))
    
    queue.subscribe(mq,
                    state.conf('tasks.list_www'),
                    state.conf('tasks.concurrent_list'),
                    queue.wrapRequestHandler(state, processTaskList))
Esempio n. 32
0
def subscribe(mq, state):
    processAuthorizeGroup = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name',
                                                                                    'group_name',
                                                                                    'port_range']),
                                                                  credentials_misc.loadCredentialForRequest,
                                                                  handleAuthorizeGroup]),
                                                 queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.authorizegroup_queue'),
                    state.conf('credentials.concurrent_authorizegroup'),
                    queue.wrapRequestHandler(state, processAuthorizeGroup))
Esempio n. 33
0
def subscribe(mq, state):
    processResumePipeline = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                    'user_name',
                                                                                    'pipeline_name']),
                                                                  pipeline_misc.forwardToCluster(state.conf,
                                                                                                 state.conf('pipelines.resume_www')),
                                                                  handleWWWResumePipeline]))
    queue.subscribe(mq,
                    state.conf('pipelines.resume_www'),
                    state.conf('pipelines.concurrent_resume'),
                    queue.wrapRequestHandler(state, processResumePipeline))
Esempio n. 34
0
def subscribe(mq, state):
    processRunInstances = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody([
                'credential_name', 'ami', 'key', 'instance_type', 'groups',
                'num_instances'
            ]), credentials_misc.loadCredentialForRequest, handleRunInstances
        ]), queue.failureMsg)
    queue.subscribe(mq, state.conf('credentials.runinstances_queue'),
                    state.conf('credentials.concurrent_runinstances'),
                    queue.wrapRequestHandler(state, processRunInstances))
Esempio n. 35
0
def subscribe(mq, state):
    processWWWDeleteCredential = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name']),
                                                                       queue.forwardRequestToCluster(
                                                                           state.conf('www.url_prefix') + '/' +
                                                                            os.path.basename(state.conf('credentials.delete_www'))),
                                                                       handleWWWDeleteCredential]),
                                                      queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.delete_www'),
                    state.conf('credentials.concurrent_deletecredential'),
                    queue.wrapRequestHandler(state, processWWWDeleteCredential))
Esempio n. 36
0
def subscribe(mq, state):
    processWWWListAddCredentials = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['cluster']),
                                                                         queue.forwardRequestToCluster(
                                                                             state.conf('www.url_prefix') + '/' +
                                                                             os.path.basename(state.conf('credentials.listaddcredentials_www'))),
                                                                         handleWWWListAddCredentials]),
                                                        queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.listaddcredentials_www'),
                    state.conf('credentials.concurrent_listaddcredentials'),
                    queue.wrapRequestHandler(state, processWWWListAddCredentials))
Esempio n. 37
0
def subscribe(mq, state):
    processListProtocols = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'user_name']),
            pipeline_misc.forwardToCluster(
                state.conf, state.conf('pipelines.listprotocols_www')),
            handleWWWListProtocols
        ]))
    queue.subscribe(mq, state.conf('pipelines.listprotocols_www'),
                    state.conf('pipelines.concurrent_listprotocols'),
                    queue.wrapRequestHandler(state, processListProtocols))
Esempio n. 38
0
def subscribe(mq, state):
    processRunPipeline = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                 'user_name',
                                                                                 'bare_run',
                                                                                 'config']),
                                                               pipeline_misc.containsPipelineTemplate,
                                                               handleWWWRunPipeline]))
    queue.subscribe(mq,
                    state.conf('pipelines.run_www'),
                    state.conf('pipelines.concurrent_run'),
                    queue.wrapRequestHandler(state, processRunPipeline))
Esempio n. 39
0
def subscribe(mq, state):
    processAddKeypair = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name',
                                                                                'keypair_name']),
                                                              credentials_misc.loadCredentialForRequest,
                                                              handleAddKeypair]),
                                             queue.failureMsg)

    queue.subscribe(mq,
                    state.conf('credentials.addkeypair_queue'),
                    state.conf('credentials.concurrent_addkeypair'),
                    queue.wrapRequestHandler(state, processAddKeypair))
Esempio n. 40
0
def subscribe(mq, state):
    processResumePipeline = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'user_name', 'pipeline_name']),
            pipeline_misc.forwardToCluster(state.conf,
                                           state.conf('pipelines.resume_www')),
            handleWWWResumePipeline
        ]))
    queue.subscribe(mq, state.conf('pipelines.resume_www'),
                    state.conf('pipelines.concurrent_resume'),
                    queue.wrapRequestHandler(state, processResumePipeline))
Esempio n. 41
0
def subscribe(mq, state):
    processTaskList = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'user_name']),
            _forwardToCluster(state.conf, state.conf('tasks.list_www')),
            handleTaskList
        ]))

    queue.subscribe(mq, state.conf('tasks.list_www'),
                    state.conf('tasks.concurrent_list'),
                    queue.wrapRequestHandler(state, processTaskList))
Esempio n. 42
0
def subscribe(mq, state):
    processValidatePipelineConfig = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                            'bare_run',
                                                                                            'config']),
                                                                          pipeline_misc.containsPipelineTemplate,
                                                                          pipeline_misc.forwardToCluster(state.conf,
                                                                                                         state.conf('pipelines.validate_www')),
                                                                          handleWWWValidatePipelineConfig]))
    queue.subscribe(mq,
                    state.conf('pipelines.validate_www'),
                    state.conf('pipelines.concurrent_validate'),
                    queue.wrapRequestHandler(state, processValidatePipelineConfig))
Esempio n. 43
0
def subscribe(mq, state):
    processUpdatePipelineConfig = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                          'user_name',
                                                                                          'criteria',
                                                                                          'config']),
                                                                        pipeline_misc.forwardToCluster(state.conf,
                                                                                                       state.conf('pipelines.update_www')),
                                                                        handleWWWUpdatePipelineConfig]))
    queue.subscribe(mq,
                    state.conf('pipelines.update_www'),
                    state.conf('pipelines.concurrent_update'),
                    queue.wrapRequestHandler(state, processUpdatePipelineConfig))
Esempio n. 44
0
def subscribe(mq, state):
    processUpdatePipelineConfig = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'user_name', 'criteria', 'config']),
            pipeline_misc.forwardToCluster(state.conf,
                                           state.conf('pipelines.update_www')),
            handleWWWUpdatePipelineConfig
        ]))
    queue.subscribe(
        mq, state.conf('pipelines.update_www'),
        state.conf('pipelines.concurrent_update'),
        queue.wrapRequestHandler(state, processUpdatePipelineConfig))
Esempio n. 45
0
def subscribe(mq, state):
    yield defer_utils.tryUntil(10,
                               lambda : _monitorAnyPipelines(mq, state),
                               onFailure=defer_utils.sleep(2))
    
    processPipelineList = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                  'user_name']),
                                                                _forwardToCluster(state.conf, state.conf('pipelines.list_www')),
                                                                handleWWWPipelineList]))
    queue.subscribe(mq,
                    state.conf('pipelines.list_www'),
                    state.conf('pipelines.concurrent_list'),
                    queue.wrapRequestHandler(state, processPipelineList))
Esempio n. 46
0
def subscribe(mq, state):
    processWWWListAddCredentials = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody(['cluster']),
            queue.forwardRequestToCluster(
                state.conf('www.url_prefix') + '/' + os.path.basename(
                    state.conf('credentials.listaddcredentials_www'))),
            handleWWWListAddCredentials
        ]), queue.failureMsg)
    queue.subscribe(
        mq, state.conf('credentials.listaddcredentials_www'),
        state.conf('credentials.concurrent_listaddcredentials'),
        queue.wrapRequestHandler(state, processWWWListAddCredentials))
Esempio n. 47
0
def subscribe(mq, state):
    processValidatePipelineConfig = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'bare_run', 'config']),
            pipeline_misc.containsPipelineTemplate,
            pipeline_misc.forwardToCluster(
                state.conf, state.conf('pipelines.validate_www')),
            handleWWWValidatePipelineConfig
        ]))
    queue.subscribe(
        mq, state.conf('pipelines.validate_www'),
        state.conf('pipelines.concurrent_validate'),
        queue.wrapRequestHandler(state, processValidatePipelineConfig))
Esempio n. 48
0
def subscribe(mq, state):
    processPipelineCreate = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                    'user_name',
                                                                                    'pipeline_name',
                                                                                    'protocol',
                                                                                    'queue',
                                                                                    'config']),
                                                                  pipeline_misc.forwardToCluster(state.conf,
                                                                                                 state.conf('pipelines.create_www')),
                                                                  handleWWWPipelineCreate]))
    queue.subscribe(mq,
                    state.conf('pipelines.create_www'),
                    state.conf('pipelines.concurrent_create'),
                    queue.wrapRequestHandler(state, processPipelineCreate))
Esempio n. 49
0
def subscribe(mq, state):
    processRunInstances = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name',
                                                                                  'ami',
                                                                                  'key',
                                                                                  'instance_type',
                                                                                  'groups',
                                                                                  'num_instances']),
                                                                credentials_misc.loadCredentialForRequest,
                                                                handleRunInstances]),
                                               queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.runinstances_queue'),
                    state.conf('credentials.concurrent_runinstances'),
                    queue.wrapRequestHandler(state, processRunInstances))
Esempio n. 50
0
def subscribe(mq, state):
    yield defer_utils.tryUntil(10,
                               lambda: _monitorAnyPipelines(mq, state),
                               onFailure=defer_utils.sleep(2))

    processPipelineList = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'user_name']),
            _forwardToCluster(state.conf, state.conf('pipelines.list_www')),
            handleWWWPipelineList
        ]))
    queue.subscribe(mq, state.conf('pipelines.list_www'),
                    state.conf('pipelines.concurrent_list'),
                    queue.wrapRequestHandler(state, processPipelineList))
Esempio n. 51
0
def subscribe(mq, state):
    processObserver = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['id',
                                                                              'file',
                                                                              'event',
                                                                              'retval',
                                                                              'props',
                                                                              'host',
                                                                              'time',
                                                                              'name',
                                                                              'message']),
                                                            handleWWWObserver]))
    queue.subscribe(mq,
                    state.conf('pipelines.observer_www'),
                    state.conf('pipelines.concurrent_observer'),
                    queue.wrapRequestHandler(state, processObserver))
Esempio n. 52
0
def subscribe(mq, state):
    createAndForward = queue.createTaskAndForward(
        state.conf('clusters.terminatecluster_queue'), 'terminateCluster', 1)
    processTerminatePipe = defer_pipe.pipe(
        [queue.keysInBody(['cluster_name', 'user_name']), createAndForward])

    processTerminateCluster = queue.returnResponse(processTerminatePipe)

    queue.subscribe(mq, state.conf('clusters.terminatecluster_www'),
                    state.conf('clusters.concurrent_terminatecluster'),
                    queue.wrapRequestHandler(state, processTerminateCluster))

    queue.subscribe(
        mq, state.conf('clusters.terminatecluster_queue'),
        state.conf('clusters.concurrent_terminatecluster'),
        queue.wrapRequestHandlerTask(state, handleTerminateCluster))
Esempio n. 53
0
 def _subscribe(self):
     processEvent = defer_pipe.pipe([queue.keysInBody(['id',
                                                       'file',
                                                       'event',
                                                       'retval',
                                                       'props',
                                                       'host',
                                                       'time',
                                                       'name',
                                                       'message']),
                                     self._handleEventMessage])
     
     queue.subscribe(self.mq,
                     self._queueName(),
                     1,
                     queue.wrapRequestHandler(None, processEvent))
Esempio n. 54
0
def subscribe(mq, state):
    processTagData = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'tag_name', 'metadata', 'action']),
            _forwardToCluster(state.conf, state.conf('tags.createupdate_www')),
            _validateAction,
            queue.createTaskAndForward(state.conf('tags.createupdate_queue'),
                                       'tagCreateUpdate', 0), _returnTag
        ]))
    queue.subscribe(mq, state.conf('tags.createupdate_www'),
                    state.conf('tags.concurrent_createupdate'),
                    queue.wrapRequestHandler(state, processTagData))

    queue.subscribe(mq, state.conf('tags.createupdate_queue'),
                    state.conf('tags.concurrent_createupdate'),
                    queue.wrapRequestHandlerTask(state, handleTaskTagData))
def subscribe(mq, state):
    processUpdateInstances = defer_pipe.hookError(
        defer_pipe.pipe(
            [
                queue.keysInBody(["credential_name", "instances"]),
                credentials_misc.loadCredentialForRequest,
                handleUpdateInstances,
            ]
        ),
        queue.failureMsg,
    )
    queue.subscribe(
        mq,
        state.conf("credentials.updateinstances_queue"),
        state.conf("credentials.concurrent_updateinstances"),
        queue.wrapRequestHandler(state, processUpdateInstances),
    )
Esempio n. 56
0
def subscribe(mq, state):
    processDeleteTag = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                               'tag_name']),
                                                             _forwardToCluster(state.conf,
                                                                               state.conf('tags.delete_www')),
                                                             queue.createTaskAndForward(state.conf('tags.delete_queue'),
                                                                                        'deleteTag',
                                                                                        0)]))

    queue.subscribe(mq,
                    state.conf('tags.delete_www'),
                    state.conf('tags.concurrent_delete'),
                    queue.wrapRequestHandler(state, processDeleteTag))

    queue.subscribe(mq,
                    state.conf('tags.delete_queue'),
                    state.conf('tags.concurrent_delete'),
                    queue.wrapRequestHandlerTask(state, handleDeleteTag))
Esempio n. 57
0
def subscribe(mq, state):
    processTransferTag = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody([
                'cluster', 'user_name', 'tag_name', 'src_cluster',
                'dst_cluster', 'dst_type'
            ]),
            _forwardToCluster(state.conf, state.conf('tags.transfer_www')),
            queue.createTaskAndForward(state.conf('tags.transfer_queue'),
                                       'transferTag', 0)
        ]))

    queue.subscribe(mq, state.conf('tags.transfer_www'),
                    state.conf('tags.concurrent_transfer'),
                    queue.wrapRequestHandler(state, processTransferTag))

    queue.subscribe(mq, state.conf('tags.transfer_queue'),
                    state.conf('tags.concurrent_transfer'),
                    queue.wrapRequestHandlerTask(state, handleTransferTag))
Esempio n. 58
0
def subscribe(mq, state):
    createAndForward = queue.createTaskAndForward(
        state.conf('clusters.startcluster_queue'), 'startCluster', 5)

    processStartPipe = defer_pipe.pipe([
        queue.keysInBody([
            'cluster_name', 'user_name', 'num_exec', 'num_data', 'cred_name',
            'conf'
        ]), returnClusterStartTaskIfExists, createCluster, createAndForward
    ])

    processStartCluster = queue.returnResponse(processStartPipe)

    queue.subscribe(mq, state.conf('clusters.startcluster_www'),
                    state.conf('clusters.concurrent_startcluster'),
                    queue.wrapRequestHandler(state, processStartCluster))

    queue.subscribe(mq, state.conf('clusters.startcluster_queue'),
                    state.conf('clusters.concurrent_startcluster'),
                    queue.wrapRequestHandlerTask(state, handleStartCluster))