コード例 #1
0
ファイル: tasklets_manager.py プロジェクト: carze/vappio
def makeService(conf):
    mqService = client.makeService(conf)
    mqFactory = mqService.mqFactory

    processRequest = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'tasklet', 'conf']),
            forwardOrCreate(
                conf('www.url_prefix') + '/' +
                os.path.basename(conf('tasklets.tasklets_www')),
                conf('tasklets.tasklets_queue'), 'runTasklets', 1)
        ]))

    queue.subscribe(mqFactory, conf('tasklets.tasklets_www'),
                    conf('tasklets.concurrent_tasklets'),
                    queue.wrapRequestHandler(None, processRequest))

    processRunTasklet = defer_pipe.hookError(
        defer_pipe.pipe(
            [queue.keysInBody(['tasklet', 'conf']), handleRunTasklet]),
        queue.failureMsg)

    queue.subscribe(mqFactory, conf('tasklets.tasklets_queue'),
                    conf('tasklets.concurrent_tasklets'),
                    queue.wrapRequestHandlerTask(None, processRunTasklet))

    return mqService
コード例 #2
0
ファイル: tasklets_manager.py プロジェクト: carze/vappio
def makeService(conf):
    mqService = client.makeService(conf)
    mqFactory = mqService.mqFactory


    processRequest = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                             'tasklet',
                                                                             'conf']),
                                                           forwardOrCreate(
                                                               conf('www.url_prefix') + '/' +
                                                               os.path.basename(conf('tasklets.tasklets_www')),
                                                               conf('tasklets.tasklets_queue'),
                                                               'runTasklets',
                                                               1)]))



    queue.subscribe(mqFactory,
                    conf('tasklets.tasklets_www'),
                    conf('tasklets.concurrent_tasklets'),
                    queue.wrapRequestHandler(None, processRequest))


    processRunTasklet = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['tasklet',
                                                                                'conf']),
                                                              handleRunTasklet]),
                                             queue.failureMsg)
    
    queue.subscribe(mqFactory,
                    conf('tasklets.tasklets_queue'),
                    conf('tasklets.concurrent_tasklets'),
                    queue.wrapRequestHandlerTask(None, processRunTasklet))
    
    
    return mqService
コード例 #3
0
def subscribe(mq, state):
    """Subscribes to the queues needed to handle any incoming import cluster 
    requests.

    """
    createAndForward = queue.createTaskAndForward(state.conf('clusters.importcluster_queue'),
                                                  'importCluster',
                                                  4)

    processImportPipe = defer_pipe.pipe([queue.keysInBody(['host',
                                                           'cred_name',
                                                           'user_name',
                                                           'src_cluster',
                                                           'dst_cluster']),
                                        returnClusterImportTaskIfExists,
                                        createCluster,
                                        createAndForward])

    processImportCluster = queue.returnResponse(processImportPipe)

    queue.subscribe(mq,
                    state.conf('clusters.importcluster_www'),
                    state.conf('clusters.concurrent_importcluster'),
                    queue.wrapRequestHandler(state, processImportCluster))

    queue.subscribe(mq,
                    state.conf('clusters.importcluster_queue'),
                    state.conf('clusters.concurrent_importcluster'),
                    queue.wrapRequestHandlerTask(state, handleImportCluster))
コード例 #4
0
def subscribe(mq, state):
    url = (state.conf('www.url_prefix') +
           '/' +
           os.path.basename(state.conf('clusters.addinstances_www')))
    forwardOrCreate = forwardOrCreateTask(url,
                                          state.conf('clusters.addinstances_queue'),
                                          'addInstances',
                                          4)

    processAddPipe = defer_pipe.pipe([queue.keysInBody(['cluster',
                                                        'user_name',
                                                        'num_exec',
                                                        'num_data']),
                                      forwardOrCreate])
    
    processAddInstances = queue.returnResponse(processAddPipe)

    queue.subscribe(mq,
                    state.conf('clusters.addinstances_www'),
                    state.conf('clusters.concurrent_addinstances'),
                    queue.wrapRequestHandler(state, processAddInstances))

    queue.subscribe(mq,
                    state.conf('clusters.addinstances_queue'),
                    state.conf('clusters.concurrent_addinstances'),
                    queue.wrapRequestHandlerTask(state, handleAddInstances))
コード例 #5
0
def subscribe(mq, state):
    processPipe = defer_pipe.pipe(
        [queue.keysInBody(['user_name']), handleList])
    processClusterList = queue.returnResponse(processPipe)
    queue.subscribe(mq, state.conf('clusters.listclusters_www'),
                    state.conf('clusters.concurrent_listclusters'),
                    queue.wrapRequestHandler(state, processClusterList))
コード例 #6
0
ファイル: cluster_mq_config.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processPipe = defer_pipe.pipe(
        [queue.keysInBody(['user_name', 'cluster_name']), handleConfig])
    processClusterConfig = queue.returnResponse(processPipe)
    queue.subscribe(mq, state.conf('clusters.config_www'),
                    state.conf('clusters.concurrent_config'),
                    queue.wrapRequestHandler(state, processClusterConfig))
コード例 #7
0
def subscribe(mq, state):
    createAndForward = queue.createTaskAndForward(state.conf('clusters.startcluster_queue'),
                                                  'startCluster',
                                                  5)

    processStartPipe = defer_pipe.pipe([queue.keysInBody(['cluster_name',
                                                          'user_name',
                                                          'num_exec',
                                                          'num_data',
                                                          'cred_name',
                                                          'conf']),
                                        returnClusterStartTaskIfExists,
                                        createCluster,
                                        createAndForward])
    
    processStartCluster = queue.returnResponse(processStartPipe)

    queue.subscribe(mq,
                    state.conf('clusters.startcluster_www'),
                    state.conf('clusters.concurrent_startcluster'),
                    queue.wrapRequestHandler(state, processStartCluster))

    queue.subscribe(mq,
                    state.conf('clusters.startcluster_queue'),
                    state.conf('clusters.concurrent_startcluster'),
                    queue.wrapRequestHandlerTask(state, handleStartCluster))
コード例 #8
0
ファイル: cluster_mq_list.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processPipe = defer_pipe.pipe([queue.keysInBody(['user_name']),
                                   handleList])
    processClusterList = queue.returnResponse(processPipe)
    queue.subscribe(mq,
                    state.conf('clusters.listclusters_www'),
                    state.conf('clusters.concurrent_listclusters'),
                    queue.wrapRequestHandler(state, processClusterList))
コード例 #9
0
ファイル: credential_mq_config.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processCredentialConfig = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody(['credential_name']),
            credentials_misc.loadCredentialForRequest, handleCredentialConfig
        ]), queue.failureMsg)
    queue.subscribe(mq, state.conf('credentials.credentialconfig_queue'),
                    state.conf('credentials.concurrent_credentialconfig'),
                    queue.wrapRequestHandler(state, processCredentialConfig))
コード例 #10
0
def subscribe(mq, state):
    processRunPipeline = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'user_name', 'bare_run', 'config']),
            pipeline_misc.containsPipelineTemplate, handleWWWRunPipeline
        ]))
    queue.subscribe(mq, state.conf('pipelines.run_www'),
                    state.conf('pipelines.concurrent_run'),
                    queue.wrapRequestHandler(state, processRunPipeline))
コード例 #11
0
def subscribe(mq, state):
    processGetCType = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody(['credential_name']),
            credentials_misc.loadCredentialForRequest, handleGetCType
        ]), queue.failureMsg)
    queue.subscribe(mq, state.conf('credentials.getctype_queue'),
                    state.conf('credentials.concurrent_getctype'),
                    queue.wrapRequestHandler(state, processGetCType))
コード例 #12
0
def subscribe(mq, state):
    processUpdateInstances = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody(['credential_name', 'instances']),
            credentials_misc.loadCredentialForRequest, handleUpdateInstances
        ]), queue.failureMsg)
    queue.subscribe(mq, state.conf('credentials.updateinstances_queue'),
                    state.conf('credentials.concurrent_updateinstances'),
                    queue.wrapRequestHandler(state, processUpdateInstances))
コード例 #13
0
def subscribe(mq, state):
    processListKeypairs = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody(['credential_name']),
            credentials_misc.loadCredentialForRequest, handleListKeypairs
        ]), queue.failureMsg)
    queue.subscribe(mq, state.conf('credentials.listkeypairs_queue'),
                    state.conf('credentials.concurrent_listkeypairs'),
                    queue.wrapRequestHandler(state, processListKeypairs))
コード例 #14
0
def subscribe(mq, state):
    processGetCType = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name']),
                                                            credentials_misc.loadCredentialForRequest,
                                                            handleGetCType]),
                                           queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.getctype_queue'),
                    state.conf('credentials.concurrent_getctype'),
                    queue.wrapRequestHandler(state, processGetCType))
コード例 #15
0
def subscribe(mq, state):
    processListKeypairs = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name']),
                                                                credentials_misc.loadCredentialForRequest,
                                                                handleListKeypairs]),
                                               queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.listkeypairs_queue'),
                    state.conf('credentials.concurrent_listkeypairs'),
                    queue.wrapRequestHandler(state, processListKeypairs))
コード例 #16
0
ファイル: cluster_mq_config.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processPipe = defer_pipe.pipe([queue.keysInBody(['user_name',
                                                     'cluster_name']),
                                   handleConfig])
    processClusterConfig = queue.returnResponse(processPipe)
    queue.subscribe(mq,
                    state.conf('clusters.config_www'),
                    state.conf('clusters.concurrent_config'),
                    queue.wrapRequestHandler(state, processClusterConfig))
コード例 #17
0
ファイル: credential_mq_config.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processCredentialConfig = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name']),
                                                                    credentials_misc.loadCredentialForRequest,
                                                                    handleCredentialConfig]),
                                                   queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.credentialconfig_queue'),
                    state.conf('credentials.concurrent_credentialconfig'),
                    queue.wrapRequestHandler(state, processCredentialConfig))
コード例 #18
0
    def _subscribe(self):
        processEvent = defer_pipe.pipe([
            queue.keysInBody([
                'id', 'file', 'event', 'retval', 'props', 'host', 'time',
                'name', 'message'
            ]), self._handleEventMessage
        ])

        queue.subscribe(self.mq, self._queueName(), 1,
                        queue.wrapRequestHandler(None, processEvent))
コード例 #19
0
def subscribe(mq, state):
    processAddGroup = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody(
                ['credential_name', 'group_name', 'group_description']),
            credentials_misc.loadCredentialForRequest, handleAddGroup
        ]), queue.failureMsg)
    queue.subscribe(mq, state.conf('credentials.addgroup_queue'),
                    state.conf('credentials.concurrent_addgroup'),
                    queue.wrapRequestHandler(state, processAddGroup))
コード例 #20
0
def subscribe(mq, state):
    processTerminateInstances = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name',
                                                                                        'instances']),
                                                                      credentials_misc.loadCredentialForRequest,
                                                                      handleTerminateInstances]),
                                                     queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.terminateinstances_queue'),
                    state.conf('credentials.concurrent_terminateinstances'),
                    queue.wrapRequestHandler(state, processTerminateInstances))
コード例 #21
0
ファイル: protocol_www_list.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processListProtocols = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                   'user_name']),
                                                                 pipeline_misc.forwardToCluster(state.conf,
                                                                                                state.conf('pipelines.listprotocols_www')),
                                                                 handleWWWListProtocols]))
    queue.subscribe(mq,
                    state.conf('pipelines.listprotocols_www'),
                    state.conf('pipelines.concurrent_listprotocols'),
                    queue.wrapRequestHandler(state, processListProtocols))
コード例 #22
0
ファイル: pipeline_www_delete.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processPipelineDelete = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                    'user_name',
                                                                                    'criteria']),
                                                                  _forwardToCluster(state.conf, state.conf('pipelines.delete_www')),
                                                                  handleWWWPipelineDelete]))
    queue.subscribe(mq,
                    state.conf('pipelines.delete_www'),
                    state.conf('pipelines.concurrent_delete'),
                    queue.wrapRequestHandler(state, processPipelineDelete))
コード例 #23
0
ファイル: pipeline_www_delete.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processPipelineDelete = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'user_name', 'criteria']),
            _forwardToCluster(state.conf, state.conf('pipelines.delete_www')),
            handleWWWPipelineDelete
        ]))
    queue.subscribe(mq, state.conf('pipelines.delete_www'),
                    state.conf('pipelines.concurrent_delete'),
                    queue.wrapRequestHandler(state, processPipelineDelete))
コード例 #24
0
def subscribe(mq, state):
    processAddKeypair = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody(['credential_name', 'keypair_name']),
            credentials_misc.loadCredentialForRequest, handleAddKeypair
        ]), queue.failureMsg)

    queue.subscribe(mq, state.conf('credentials.addkeypair_queue'),
                    state.conf('credentials.concurrent_addkeypair'),
                    queue.wrapRequestHandler(state, processAddKeypair))
コード例 #25
0
ファイル: pipeline_www_run.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processRunPipeline = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                 'user_name',
                                                                                 'bare_run',
                                                                                 'config']),
                                                               pipeline_misc.containsPipelineTemplate,
                                                               handleWWWRunPipeline]))
    queue.subscribe(mq,
                    state.conf('pipelines.run_www'),
                    state.conf('pipelines.concurrent_run'),
                    queue.wrapRequestHandler(state, processRunPipeline))
コード例 #26
0
def subscribe(mq, state):
    processListProtocols = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'user_name']),
            pipeline_misc.forwardToCluster(
                state.conf, state.conf('pipelines.listprotocols_www')),
            handleWWWListProtocols
        ]))
    queue.subscribe(mq, state.conf('pipelines.listprotocols_www'),
                    state.conf('pipelines.concurrent_listprotocols'),
                    queue.wrapRequestHandler(state, processListProtocols))
コード例 #27
0
ファイル: pipeline_www_resume.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processResumePipeline = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                    'user_name',
                                                                                    'pipeline_name']),
                                                                  pipeline_misc.forwardToCluster(state.conf,
                                                                                                 state.conf('pipelines.resume_www')),
                                                                  handleWWWResumePipeline]))
    queue.subscribe(mq,
                    state.conf('pipelines.resume_www'),
                    state.conf('pipelines.concurrent_resume'),
                    queue.wrapRequestHandler(state, processResumePipeline))
コード例 #28
0
def subscribe(mq, state):
    processAuthorizeGroup = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name',
                                                                                    'group_name',
                                                                                    'port_range']),
                                                                  credentials_misc.loadCredentialForRequest,
                                                                  handleAuthorizeGroup]),
                                                 queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.authorizegroup_queue'),
                    state.conf('credentials.concurrent_authorizegroup'),
                    queue.wrapRequestHandler(state, processAuthorizeGroup))
コード例 #29
0
def subscribe(mq, state):
    processAddKeypair = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name',
                                                                                'keypair_name']),
                                                              credentials_misc.loadCredentialForRequest,
                                                              handleAddKeypair]),
                                             queue.failureMsg)

    queue.subscribe(mq,
                    state.conf('credentials.addkeypair_queue'),
                    state.conf('credentials.concurrent_addkeypair'),
                    queue.wrapRequestHandler(state, processAddKeypair))
コード例 #30
0
def subscribe(mq, state):
    processWWWDeleteCredential = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name']),
                                                                       queue.forwardRequestToCluster(
                                                                           state.conf('www.url_prefix') + '/' +
                                                                            os.path.basename(state.conf('credentials.delete_www'))),
                                                                       handleWWWDeleteCredential]),
                                                      queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.delete_www'),
                    state.conf('credentials.concurrent_deletecredential'),
                    queue.wrapRequestHandler(state, processWWWDeleteCredential))
コード例 #31
0
ファイル: pipeline_www_resume.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processResumePipeline = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'user_name', 'pipeline_name']),
            pipeline_misc.forwardToCluster(state.conf,
                                           state.conf('pipelines.resume_www')),
            handleWWWResumePipeline
        ]))
    queue.subscribe(mq, state.conf('pipelines.resume_www'),
                    state.conf('pipelines.concurrent_resume'),
                    queue.wrapRequestHandler(state, processResumePipeline))
コード例 #32
0
ファイル: credential_mq_listadd.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processWWWListAddCredentials = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['cluster']),
                                                                         queue.forwardRequestToCluster(
                                                                             state.conf('www.url_prefix') + '/' +
                                                                             os.path.basename(state.conf('credentials.listaddcredentials_www'))),
                                                                         handleWWWListAddCredentials]),
                                                        queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.listaddcredentials_www'),
                    state.conf('credentials.concurrent_listaddcredentials'),
                    queue.wrapRequestHandler(state, processWWWListAddCredentials))
コード例 #33
0
ファイル: task_list.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processTaskList = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'user_name']),
            _forwardToCluster(state.conf, state.conf('tasks.list_www')),
            handleTaskList
        ]))

    queue.subscribe(mq, state.conf('tasks.list_www'),
                    state.conf('tasks.concurrent_list'),
                    queue.wrapRequestHandler(state, processTaskList))
コード例 #34
0
ファイル: task_list.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processTaskList = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                              'user_name']),
                                                            _forwardToCluster(state.conf,
                                                                              state.conf('tasks.list_www')),
                                                            handleTaskList]))
    
    queue.subscribe(mq,
                    state.conf('tasks.list_www'),
                    state.conf('tasks.concurrent_list'),
                    queue.wrapRequestHandler(state, processTaskList))
コード例 #35
0
ファイル: pipeline_www_observer.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processObserver = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody([
                'id', 'file', 'event', 'retval', 'props', 'host', 'time',
                'name', 'message'
            ]), handleWWWObserver
        ]))
    queue.subscribe(mq, state.conf('pipelines.observer_www'),
                    state.conf('pipelines.concurrent_observer'),
                    queue.wrapRequestHandler(state, processObserver))
コード例 #36
0
def subscribe(mq, state):
    processRunInstances = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody([
                'credential_name', 'ami', 'key', 'instance_type', 'groups',
                'num_instances'
            ]), credentials_misc.loadCredentialForRequest, handleRunInstances
        ]), queue.failureMsg)
    queue.subscribe(mq, state.conf('credentials.runinstances_queue'),
                    state.conf('credentials.concurrent_runinstances'),
                    queue.wrapRequestHandler(state, processRunInstances))
コード例 #37
0
def subscribe(mq, state):
    processUpdatePipelineConfig = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'user_name', 'criteria', 'config']),
            pipeline_misc.forwardToCluster(state.conf,
                                           state.conf('pipelines.update_www')),
            handleWWWUpdatePipelineConfig
        ]))
    queue.subscribe(
        mq, state.conf('pipelines.update_www'),
        state.conf('pipelines.concurrent_update'),
        queue.wrapRequestHandler(state, processUpdatePipelineConfig))
コード例 #38
0
ファイル: pipeline_www_validate.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processValidatePipelineConfig = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                            'bare_run',
                                                                                            'config']),
                                                                          pipeline_misc.containsPipelineTemplate,
                                                                          pipeline_misc.forwardToCluster(state.conf,
                                                                                                         state.conf('pipelines.validate_www')),
                                                                          handleWWWValidatePipelineConfig]))
    queue.subscribe(mq,
                    state.conf('pipelines.validate_www'),
                    state.conf('pipelines.concurrent_validate'),
                    queue.wrapRequestHandler(state, processValidatePipelineConfig))
コード例 #39
0
ファイル: pipeline_www_update.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processUpdatePipelineConfig = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                          'user_name',
                                                                                          'criteria',
                                                                                          'config']),
                                                                        pipeline_misc.forwardToCluster(state.conf,
                                                                                                       state.conf('pipelines.update_www')),
                                                                        handleWWWUpdatePipelineConfig]))
    queue.subscribe(mq,
                    state.conf('pipelines.update_www'),
                    state.conf('pipelines.concurrent_update'),
                    queue.wrapRequestHandler(state, processUpdatePipelineConfig))
コード例 #40
0
ファイル: pipeline_www_list.py プロジェクト: carze/vappio
def subscribe(mq, state):
    yield defer_utils.tryUntil(10,
                               lambda : _monitorAnyPipelines(mq, state),
                               onFailure=defer_utils.sleep(2))
    
    processPipelineList = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                  'user_name']),
                                                                _forwardToCluster(state.conf, state.conf('pipelines.list_www')),
                                                                handleWWWPipelineList]))
    queue.subscribe(mq,
                    state.conf('pipelines.list_www'),
                    state.conf('pipelines.concurrent_list'),
                    queue.wrapRequestHandler(state, processPipelineList))
コード例 #41
0
ファイル: credential_mq_listadd.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processWWWListAddCredentials = defer_pipe.hookError(
        defer_pipe.pipe([
            queue.keysInBody(['cluster']),
            queue.forwardRequestToCluster(
                state.conf('www.url_prefix') + '/' + os.path.basename(
                    state.conf('credentials.listaddcredentials_www'))),
            handleWWWListAddCredentials
        ]), queue.failureMsg)
    queue.subscribe(
        mq, state.conf('credentials.listaddcredentials_www'),
        state.conf('credentials.concurrent_listaddcredentials'),
        queue.wrapRequestHandler(state, processWWWListAddCredentials))
コード例 #42
0
def subscribe(mq, state):
    processValidatePipelineConfig = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'bare_run', 'config']),
            pipeline_misc.containsPipelineTemplate,
            pipeline_misc.forwardToCluster(
                state.conf, state.conf('pipelines.validate_www')),
            handleWWWValidatePipelineConfig
        ]))
    queue.subscribe(
        mq, state.conf('pipelines.validate_www'),
        state.conf('pipelines.concurrent_validate'),
        queue.wrapRequestHandler(state, processValidatePipelineConfig))
コード例 #43
0
ファイル: pipeline_www_list.py プロジェクト: carze/vappio
def subscribe(mq, state):
    yield defer_utils.tryUntil(10,
                               lambda: _monitorAnyPipelines(mq, state),
                               onFailure=defer_utils.sleep(2))

    processPipelineList = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'user_name']),
            _forwardToCluster(state.conf, state.conf('pipelines.list_www')),
            handleWWWPipelineList
        ]))
    queue.subscribe(mq, state.conf('pipelines.list_www'),
                    state.conf('pipelines.concurrent_list'),
                    queue.wrapRequestHandler(state, processPipelineList))
コード例 #44
0
def subscribe(mq, state):
    processRunInstances = defer_pipe.hookError(defer_pipe.pipe([queue.keysInBody(['credential_name',
                                                                                  'ami',
                                                                                  'key',
                                                                                  'instance_type',
                                                                                  'groups',
                                                                                  'num_instances']),
                                                                credentials_misc.loadCredentialForRequest,
                                                                handleRunInstances]),
                                               queue.failureMsg)
    queue.subscribe(mq,
                    state.conf('credentials.runinstances_queue'),
                    state.conf('credentials.concurrent_runinstances'),
                    queue.wrapRequestHandler(state, processRunInstances))
コード例 #45
0
def subscribe(mq, state):
    processPipelineCreate = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                    'user_name',
                                                                                    'pipeline_name',
                                                                                    'protocol',
                                                                                    'queue',
                                                                                    'config']),
                                                                  pipeline_misc.forwardToCluster(state.conf,
                                                                                                 state.conf('pipelines.create_www')),
                                                                  handleWWWPipelineCreate]))
    queue.subscribe(mq,
                    state.conf('pipelines.create_www'),
                    state.conf('pipelines.concurrent_create'),
                    queue.wrapRequestHandler(state, processPipelineCreate))
コード例 #46
0
ファイル: pipeline_www_observer.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processObserver = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['id',
                                                                              'file',
                                                                              'event',
                                                                              'retval',
                                                                              'props',
                                                                              'host',
                                                                              'time',
                                                                              'name',
                                                                              'message']),
                                                            handleWWWObserver]))
    queue.subscribe(mq,
                    state.conf('pipelines.observer_www'),
                    state.conf('pipelines.concurrent_observer'),
                    queue.wrapRequestHandler(state, processObserver))
コード例 #47
0
ファイル: pipeline_monitor.py プロジェクト: carze/vappio
 def _subscribe(self):
     processEvent = defer_pipe.pipe([queue.keysInBody(['id',
                                                       'file',
                                                       'event',
                                                       'retval',
                                                       'props',
                                                       'host',
                                                       'time',
                                                       'name',
                                                       'message']),
                                     self._handleEventMessage])
     
     queue.subscribe(self.mq,
                     self._queueName(),
                     1,
                     queue.wrapRequestHandler(None, processEvent))
コード例 #48
0
ファイル: cluster_mq_terminate.py プロジェクト: carze/vappio
def subscribe(mq, state):
    createAndForward = queue.createTaskAndForward(
        state.conf('clusters.terminatecluster_queue'), 'terminateCluster', 1)
    processTerminatePipe = defer_pipe.pipe(
        [queue.keysInBody(['cluster_name', 'user_name']), createAndForward])

    processTerminateCluster = queue.returnResponse(processTerminatePipe)

    queue.subscribe(mq, state.conf('clusters.terminatecluster_www'),
                    state.conf('clusters.concurrent_terminatecluster'),
                    queue.wrapRequestHandler(state, processTerminateCluster))

    queue.subscribe(
        mq, state.conf('clusters.terminatecluster_queue'),
        state.conf('clusters.concurrent_terminatecluster'),
        queue.wrapRequestHandlerTask(state, handleTerminateCluster))
コード例 #49
0
def subscribe(mq, state):
    processTagData = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody(['cluster', 'tag_name', 'metadata', 'action']),
            _forwardToCluster(state.conf, state.conf('tags.createupdate_www')),
            _validateAction,
            queue.createTaskAndForward(state.conf('tags.createupdate_queue'),
                                       'tagCreateUpdate', 0), _returnTag
        ]))
    queue.subscribe(mq, state.conf('tags.createupdate_www'),
                    state.conf('tags.concurrent_createupdate'),
                    queue.wrapRequestHandler(state, processTagData))

    queue.subscribe(mq, state.conf('tags.createupdate_queue'),
                    state.conf('tags.concurrent_createupdate'),
                    queue.wrapRequestHandlerTask(state, handleTaskTagData))
コード例 #50
0
def subscribe(mq, state):
    processUpdateInstances = defer_pipe.hookError(
        defer_pipe.pipe(
            [
                queue.keysInBody(["credential_name", "instances"]),
                credentials_misc.loadCredentialForRequest,
                handleUpdateInstances,
            ]
        ),
        queue.failureMsg,
    )
    queue.subscribe(
        mq,
        state.conf("credentials.updateinstances_queue"),
        state.conf("credentials.concurrent_updateinstances"),
        queue.wrapRequestHandler(state, processUpdateInstances),
    )
コード例 #51
0
ファイル: tag_mq_delete.py プロジェクト: carze/vappio
def subscribe(mq, state):
    processDeleteTag = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                               'tag_name']),
                                                             _forwardToCluster(state.conf,
                                                                               state.conf('tags.delete_www')),
                                                             queue.createTaskAndForward(state.conf('tags.delete_queue'),
                                                                                        'deleteTag',
                                                                                        0)]))

    queue.subscribe(mq,
                    state.conf('tags.delete_www'),
                    state.conf('tags.concurrent_delete'),
                    queue.wrapRequestHandler(state, processDeleteTag))

    queue.subscribe(mq,
                    state.conf('tags.delete_queue'),
                    state.conf('tags.concurrent_delete'),
                    queue.wrapRequestHandlerTask(state, handleDeleteTag))
コード例 #52
0
def subscribe(mq, state):
    processTransferTag = queue.returnResponse(
        defer_pipe.pipe([
            queue.keysInBody([
                'cluster', 'user_name', 'tag_name', 'src_cluster',
                'dst_cluster', 'dst_type'
            ]),
            _forwardToCluster(state.conf, state.conf('tags.transfer_www')),
            queue.createTaskAndForward(state.conf('tags.transfer_queue'),
                                       'transferTag', 0)
        ]))

    queue.subscribe(mq, state.conf('tags.transfer_www'),
                    state.conf('tags.concurrent_transfer'),
                    queue.wrapRequestHandler(state, processTransferTag))

    queue.subscribe(mq, state.conf('tags.transfer_queue'),
                    state.conf('tags.concurrent_transfer'),
                    queue.wrapRequestHandlerTask(state, handleTransferTag))
コード例 #53
0
ファイル: cluster_mq_terminate.py プロジェクト: carze/vappio
def subscribe(mq, state):
    createAndForward = queue.createTaskAndForward(state.conf("clusters.terminatecluster_queue"), "terminateCluster", 1)
    processTerminatePipe = defer_pipe.pipe([queue.keysInBody(["cluster_name", "user_name"]), createAndForward])

    processTerminateCluster = queue.returnResponse(processTerminatePipe)

    queue.subscribe(
        mq,
        state.conf("clusters.terminatecluster_www"),
        state.conf("clusters.concurrent_terminatecluster"),
        queue.wrapRequestHandler(state, processTerminateCluster),
    )

    queue.subscribe(
        mq,
        state.conf("clusters.terminatecluster_queue"),
        state.conf("clusters.concurrent_terminatecluster"),
        queue.wrapRequestHandlerTask(state, handleTerminateCluster),
    )
コード例 #54
0
def subscribe(mq, state):
    url = (state.conf('www.url_prefix') + '/' +
           os.path.basename(state.conf('clusters.addinstances_www')))
    forwardOrCreate = forwardOrCreateTask(
        url, state.conf('clusters.addinstances_queue'), 'addInstances', 4)

    processAddPipe = defer_pipe.pipe([
        queue.keysInBody(['cluster', 'user_name', 'num_exec', 'num_data']),
        forwardOrCreate
    ])

    processAddInstances = queue.returnResponse(processAddPipe)

    queue.subscribe(mq, state.conf('clusters.addinstances_www'),
                    state.conf('clusters.concurrent_addinstances'),
                    queue.wrapRequestHandler(state, processAddInstances))

    queue.subscribe(mq, state.conf('clusters.addinstances_queue'),
                    state.conf('clusters.concurrent_addinstances'),
                    queue.wrapRequestHandlerTask(state, handleAddInstances))
コード例 #55
0
def subscribe(mq, state):
    processRealizePhantom = queue.returnResponse(defer_pipe.pipe([queue.keysInBody(['cluster',
                                                                                    'tag_name',
                                                                                    'user_name',
                                                                                    'phantom',
                                                                                    'metadata']),
                                                                  _forwardToCluster(state.conf,
                                                                                    state.conf('tags.realize_www')),
                                                                  queue.createTaskAndForward(state.conf('tags.realize_queue'),
                                                                                             'realizePhantom',
                                                                                             0)]))
    queue.subscribe(mq,
                    state.conf('tags.realize_www'),
                    state.conf('tags.concurrent_realize'),
                    queue.wrapRequestHandler(state, processRealizePhantom))

    queue.subscribe(mq,
                    state.conf('tags.realize_queue'),
                    state.conf('tags.concurrent_realize'),
                    queue.wrapRequestHandlerTask(state, handleRealizePhantom))
コード例 #56
0
def subscribe(mq, state):
    createAndForward = queue.createTaskAndForward(
        state.conf('clusters.startcluster_queue'), 'startCluster', 5)

    processStartPipe = defer_pipe.pipe([
        queue.keysInBody([
            'cluster_name', 'user_name', 'num_exec', 'num_data', 'cred_name',
            'conf'
        ]), returnClusterStartTaskIfExists, createCluster, createAndForward
    ])

    processStartCluster = queue.returnResponse(processStartPipe)

    queue.subscribe(mq, state.conf('clusters.startcluster_www'),
                    state.conf('clusters.concurrent_startcluster'),
                    queue.wrapRequestHandler(state, processStartCluster))

    queue.subscribe(mq, state.conf('clusters.startcluster_queue'),
                    state.conf('clusters.concurrent_startcluster'),
                    queue.wrapRequestHandlerTask(state, handleStartCluster))
コード例 #57
0
def subscribe(mq, state):
    createAndForward = queue.createTaskAndForward(
        state.conf('clusters.terminateinstances_queue'), 'terminateInstances',
        1)
    processTerminateInstancesPipe = defer_pipe.pipe([
        queue.keysInBody(
            ['cluster_name', 'user_name', 'by_attribute', 'attribute_values']),
        createAndForward
    ])

    processTerminateInstances = queue.returnResponse(
        processTerminateInstancesPipe)

    queue.subscribe(mq, state.conf('clusters.terminateinstances_www'),
                    state.conf('clusters.concurrent_terminateinstances'),
                    queue.wrapRequestHandler(state, processTerminateInstances))

    queue.subscribe(
        mq, state.conf('clusters.terminateinstances_queue'),
        state.conf('clusters.concurrent_terminateinstances'),
        queue.wrapRequestHandlerTask(state, handleTerminateInstances))