def handleWWWPipelineCreate(request): """ Sets the config section of a pipeline exactly to what is given Input: { cluster: string user_name: string pipeline_name : string protocol : string queue : string config: { key/value } ?pipeline_id : string ?task_name : string ?pipeline_parent : pipeline_name } Output: Pipeline """ if request.body.get('parent_pipeline'): parentPipelines = yield request.state.pipelinePersist.loadAllPipelinesBy({'pipeline_name': request.body['parent_pipeline']}, request.body['user_name']) if len(parentPipelines) == 1: parentPipeline = parentPipelines[0] else: raise Exception('More than one possible parent pipeline choice, not sure what to do here') else: parentPipeline = None if 'task_name' in request.body: taskName = request.body['task_name'] else: taskName = yield tasks.createTaskAndSave('runPipelines', 0) pipeline = persist.Pipeline(pipelineId=request.body.get('pipeline_id', None), pipelineName=request.body['pipeline_name'], userName=request.body['user_name'], protocol=request.body['protocol'], checksum=pipeline_misc.checksumInput(request.body['config']), taskName=taskName, queue=request.body['queue'], children=[], config=request.body['config']) yield request.state.pipelinePersist.savePipeline(pipeline) pipelineDict = yield request.state.pipelinesCache.pipelineToDict(pipeline) if parentPipeline: childPipeline = [('local', request.body['pipeline_name'])] parentPipeline = parentPipeline.update(children=list(set([tuple(e) for e in parentPipeline.children + childPipeline]))) yield request.state.pipelinePersist.savePipeline(parentPipeline) defer.returnValue(request.update(response=pipelineDict))
def _run(state, batchState): if 'pipeline_name' in batchState: pipelines = yield pipelines_client.pipelineList('localhost', 'local', 'guest', batchState['pipeline_name'], detail=True) else: pipelines = [] if not pipelines: _log(batchState, 'First time running, creating pipeline state information') batchState['pipeline_config'] = yield _applyActions(state.innerPipelineConfig(), batchState['actions']) batchState['pipeline_state'] = STARTCLUSTER_STATE # We need to create a fake, local, pipeline for metrics to work batchState['pipeline_name'] = pipeline_misc.checksumInput(batchState['pipeline_config']) batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState['pipeline_name'] batchState['pipeline_config']['pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name'] _log(batchState, 'Pipeline named ' + batchState['pipeline_name']) pipeline = yield pipelines_client.createPipeline(host='localhost', clusterName='local', userName='******', pipelineName=batchState['pipeline_name'], protocol='clovr_wrapper', queue='pipeline.q', config=batchState['pipeline_config'], parentPipeline=state.parentPipeline()) batchState['lgt_wrapper_task_name'] = pipeline['task_name'] _log(batchState, 'Setting number of tasks to 9 (number in a standard lgt_wrapper)') yield _updateTask(batchState, lambda t : t.update(completedTasks=0, numTasks=9)) state.updateBatchState() batchState['state'] = RUNNING_STATE _log(batchState, 'Pipeline started in %s state' % batchState['pipeline_state']) yield _updateTask(batchState, lambda t : t.setState(tasks.task.TASK_RUNNING)) pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs', global_state.make_ref() + '.conf') _log(batchState, 'Creating ergatis configuration') _writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile) if batchState['pipeline_state'] == STARTCLUSTER_STATE: _log(batchState, 'Pipeline is in STARTCLUSTER state') # First see if the cluster exists but is unresponsive try: cluster = yield loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, terminating') terminateTask = yield clusters_client.terminateCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _blockOnTask(terminateTask) except: pass batchState['cluster_task'] = yield startCluster( batchState, 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', int(batchState['pipeline_config']['cluster.EXEC_NODES']), 0, batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'], {'cluster.MASTER_INSTANCE_TYPE': batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'], 'cluster.MASTER_BID_PRICE': batchState['pipeline_config']['cluster.MASTER_BID_PRICE'], 'cluster.EXEC_INSTANCE_TYPE': batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'], 'cluster.EXEC_BID_PRICE': batchState['pipeline_config']['cluster.EXEC_BID_PRICE']}) yield _updateTask(batchState, lambda t : t.update(completedTasks=0, numTasks=9)) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed startcluster' ).progress()) batchState['pipeline_state'] = REMOTE_LOCAL_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REMOTE_LOCAL_TRANSFER_STATE: _log(batchState, 'Pipeline is in REMOTE_LOCAL_TRANSFER') _log(batchState, 'Making sure cluster is exists in some form') cluster = yield clusters_client.loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, erroring and restarting') raise Exception('Cluster is not responsive') yield state.prerunQueue.addWithDeferred(_remoteLocalTransfer, batchState) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed remote_local_transfer' ).progress()) batchState['pipeline_state'] = DECRYPT_STATE state.updateBatchState() if batchState['pipeline_state'] == DECRYPT_STATE: _log(batchState, 'Pipeline is in DECRYPT') cluster = yield loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') tag = yield tags_client.loadTag('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', _decryptTagName(batchState)) conf = config.configFromStream(open('/tmp/machine.conf')) yield ssh.runProcessSSH(cluster['master']['public_dns'], 'mkdir -p /mnt/lgt_decrypt', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) yield rsync.rsyncTo(cluster['master']['public_dns'], batchState['pipeline_config']['params.DECRYPT_SCRIPT'], '/mnt/', options=conf('rsync.options'), user=conf('rsync.user'), log=True) for f in tag['files']: decryptCmd = ' '.join([os.path.join('/mnt', os.path.basename(batchState['pipeline_config']['params.DECRYPT_SCRIPT'])), f, '-out-dir', '/mnt/lgt_decrypt', '-remove-encrypted', '-password', batchState['pipeline_config']['params.DECRYPT_PASSWORD']]) yield ssh.getOutput(cluster['master']['public_dns'], decryptCmd, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), expected=[0, 253], log=True) tag = yield tags_client.tagData(host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', action='overwrite', tagName=_decryptTagName(batchState), files=['/mnt/lgt_decrypt'], metadata={}, recursive=True, expand=False, compressDir=None) _log(batchState, 'Waiting for tagging of %s to complete - %s' % (_decryptTagName(batchState), tag['task_name'])) yield _blockOnTask(tag['task_name'], cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME']) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed decrypt' ).progress()) batchState['pipeline_state'] = REFERENCE_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REFERENCE_TRANSFER_STATE: _log(batchState, 'Pipeline is in REFERENCE_TRANSFER state') transfers = [] tags = (batchState['pipeline_config']['input.REF_TAG1'].split(',') + batchState['pipeline_config']['input.REF_TAG2'].split(',')) for tag in tags: tag = tag.strip() output = yield _getOutput(batchState, ['vp-transfer-dataset', '-t', '--tag-name=' + tag, '--dst-cluster=' + batchState['pipeline_config']['cluster.CLUSTER_NAME']], log=True) transfers.append(output['stdout'].strip()) for task in transfers: yield _blockOnTask(task) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed reference_transfer' ).progress()) batchState['pipeline_state'] = RUN_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUN_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUN_PIPELINE state') batchState['pipeline_config']['input.INPUT_TAG'] = _decryptTagName(batchState) pipeline = yield pipelines_client.runPipeline(host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', parentPipeline=batchState['pipeline_name'], bareRun=True, queue=state.innerPipelineQueue(), config=batchState['pipeline_config'], overwrite=True) batchState['pipeline_task'] = pipeline['task_name'] yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed run pipeline' ).progress()) batchState['pipeline_state'] = RUNNING_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUNNING_PIPELINE state') _monitorPipeline(batchState) yield _waitForPipeline(batchState) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed running pipeline' ).progress()) batchState['pipeline_state'] = HARVEST_STATE state.updateBatchState() if batchState['pipeline_state'] == HARVEST_STATE: _log(batchState, 'Pipeline is in HARVEST state') # Using prerunqueue because we want everything here serialized yield state.prerunQueue.addWithDeferred(_harvestTransfer, batchState) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed harvest' ).progress()) batchState['pipeline_state'] = SHUTDOWN_STATE state.updateBatchState() if batchState['pipeline_state'] == SHUTDOWN_STATE: _log(batchState, 'Pipeline is in SHUTDOWN state') if 'add_instances_task' in batchState: try: yield _blockOnTask(batchState['add_instances_task'], cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME']) except Exception, err: logging.errorPrint(str(err)) log.err(err) yield clusters_client.terminateCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed shutdown' ).progress()) batchState['pipeline_state'] = COMPLETED_STATE batchState['state'] = COMPLETED_STATE state.updateBatchState()
def _run(state, batchState): if 'state' not in batchState: _log(batchState, 'First time running, creating pipeline state information') batchState['pipeline_config'] = yield _applyActions( state.innerPipelineConfig(), batchState['actions']) batchState['pipeline_state'] = PRESTART_STATE # We need to create a fake, local, pipeline for metrics to work batchState['pipeline_name'] = pipeline_misc.checksumInput( batchState['pipeline_config']) batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState[ 'pipeline_name'] batchState['pipeline_config'][ 'pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name'] _log(batchState, 'Pipeline named ' + batchState['pipeline_name']) pipeline = yield pipelines_client.createPipeline( host='localhost', clusterName='local', userName='******', pipelineName=batchState['pipeline_name'], protocol='clovr_wrapper', queue='pipeline.q', config=batchState['pipeline_config'], parentPipeline=state.parentPipeline()) batchState['clovr_wrapper_task_name'] = pipeline['task_name'] _log( batchState, 'Setting number of tasks to 6 (number in a standard clovr_wrapper)' ) yield _updateTask(batchState, lambda t: t.update(completedTasks=0, numTasks=6)) state.updateBatchState() else: _log(batchState, 'Pipeline run before, loading pipeline information') pipeline = yield pipelines_client.pipelineList( 'localhost', 'local', 'guest', batchState['pipeline_name'], detail=True) batchState['state'] = RUNNING_STATE yield _updateTask(batchState, lambda t: t.setState(tasks.task.TASK_RUNNING)) pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs', global_state.make_ref() + '.conf') _log(batchState, 'Creating ergatis configuration') _writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile) if batchState['pipeline_state'] == PRESTART_STATE: _log(batchState, 'Pipeline is in PRESTART state') yield state.prerunQueue.addWithDeferred( workflow_runner.run, state.workflowConfig(), batchState['pipeline_config']['pipeline.PRESTART_TEMPLATE_XML'], pipelineConfigFile, TMP_DIR) yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed prestart').progress()) batchState['pipeline_state'] = STARTING_STATE state.updateBatchState() if batchState['pipeline_state'] == STARTING_STATE: _log(batchState, 'Pipeline is in STARTING state') clusterTask = yield clusters_client.startCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', int(batchState['pipeline_config']['cluster.EXEC_NODES']), 0, batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'], { 'cluster.master_type': batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'], 'cluster.master_bid_price': batchState['pipeline_config']['cluster.MASTER_BID_PRICE'], 'cluster.exec_type': batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'], 'cluster.exec_bid_price': batchState['pipeline_config']['cluster.EXEC_BID_PRICE'] }) taskState = yield tasks.blockOnTask('localhost', 'local', clusterTask) if taskState != tasks.task.TASK_COMPLETED: raise TaskError(clusterTask) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed start').progress()) batchState['pipeline_state'] = PRERUN_STATE state.updateBatchState() if batchState['pipeline_state'] == PRERUN_STATE: _log(batchState, 'Pipeline is in PRERUN state') yield state.prerunQueue.addWithDeferred( workflow_runner.run, state.workflowConfig(), batchState['pipeline_config']['pipeline.PRERUN_TEMPLATE_XML'], pipelineConfigFile, TMP_DIR) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed prerun').progress()) batchState['pipeline_state'] = RUN_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUN_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUN_PIPELINE state') pipeline = yield pipelines_client.runPipeline( host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', parentPipeline=batchState['pipeline_name'], bareRun=True, queue=state.innerPipelineQueue(), config=batchState['pipeline_config'], overwrite=True) batchState['pipeline_task'] = pipeline['task_name'] yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed run pipeline').progress()) batchState['pipeline_state'] = RUNNING_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUNNING_PIPELINE state') _monitorPipeline(batchState) yield _waitForPipeline(batchState) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed running pipeline').progress()) batchState['pipeline_state'] = POSTRUN_STATE state.updateBatchState() if batchState['pipeline_state'] == POSTRUN_STATE: _log(batchState, 'Pipeline is in POSTRUN state') yield state.postrunQueue.addWithDeferred( workflow_runner.run, state.workflowConfig(), batchState['pipeline_config']['pipeline.POSTRUN_TEMPLATE_XML'], pipelineConfigFile, TMP_DIR) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed postrun').progress()) batchState['pipeline_state'] = COMPLETED_STATE batchState['state'] = COMPLETED_STATE state.updateBatchState() yield _updateTask(batchState, lambda t: t.setState(tasks.task.TASK_COMPLETED)) _log(batchState, 'Pipeline finished successfully')
def _run(state, batchState): if 'state' not in batchState: _log(batchState, 'First time running, creating pipeline state information') batchState['pipeline_config'] = yield _applyActions(state.innerPipelineConfig(), batchState['actions']) batchState['pipeline_state'] = PRESTART_STATE # We need to create a fake, local, pipeline for metrics to work batchState['pipeline_name'] = pipeline_misc.checksumInput(batchState['pipeline_config']) batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState['pipeline_name'] batchState['pipeline_config']['pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name'] _log(batchState, 'Pipeline named ' + batchState['pipeline_name']) pipeline = yield pipelines_client.createPipeline(host='localhost', clusterName='local', userName='******', pipelineName=batchState['pipeline_name'], protocol='clovr_wrapper', queue='pipeline.q', config=batchState['pipeline_config'], parentPipeline=state.parentPipeline()) batchState['clovr_wrapper_task_name'] = pipeline['task_name'] _log(batchState, 'Setting number of tasks to 6 (number in a standard clovr_wrapper)') yield _updateTask(batchState, lambda t : t.update(completedTasks=0, numTasks=6)) state.updateBatchState() else: _log(batchState, 'Pipeline run before, loading pipeline information') pipeline = yield pipelines_client.pipelineList('localhost', 'local', 'guest', batchState['pipeline_name'], detail=True) batchState['state'] = RUNNING_STATE yield _updateTask(batchState, lambda t : t.setState(tasks.task.TASK_RUNNING)) pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs', global_state.make_ref() + '.conf') _log(batchState, 'Creating ergatis configuration') _writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile) if batchState['pipeline_state'] == PRESTART_STATE: _log(batchState, 'Pipeline is in PRESTART state') yield state.prerunQueue.addWithDeferred(workflow_runner.run, state.workflowConfig(), batchState['pipeline_config']['pipeline.PRESTART_TEMPLATE_XML'], pipelineConfigFile, TMP_DIR) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed prestart' ).progress()) batchState['pipeline_state'] = STARTING_STATE state.updateBatchState() if batchState['pipeline_state'] == STARTING_STATE: _log(batchState, 'Pipeline is in STARTING state') clusterTask = yield clusters_client.startCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', int(batchState['pipeline_config']['cluster.EXEC_NODES']), 0, batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'], {'cluster.master_type': batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'], 'cluster.master_bid_price': batchState['pipeline_config']['cluster.MASTER_BID_PRICE'], 'cluster.exec_type': batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'], 'cluster.exec_bid_price': batchState['pipeline_config']['cluster.EXEC_BID_PRICE']}) taskState = yield tasks.blockOnTask('localhost', 'local', clusterTask) if taskState != tasks.task.TASK_COMPLETED: raise TaskError(clusterTask) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed start' ).progress()) batchState['pipeline_state'] = PRERUN_STATE state.updateBatchState() if batchState['pipeline_state'] == PRERUN_STATE: _log(batchState, 'Pipeline is in PRERUN state') yield state.prerunQueue.addWithDeferred(workflow_runner.run, state.workflowConfig(), batchState['pipeline_config']['pipeline.PRERUN_TEMPLATE_XML'], pipelineConfigFile, TMP_DIR) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed prerun' ).progress()) batchState['pipeline_state'] = RUN_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUN_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUN_PIPELINE state') pipeline = yield pipelines_client.runPipeline(host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', parentPipeline=batchState['pipeline_name'], bareRun=True, queue=state.innerPipelineQueue(), config=batchState['pipeline_config'], overwrite=True) batchState['pipeline_task'] = pipeline['task_name'] yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed run pipeline' ).progress()) batchState['pipeline_state'] = RUNNING_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUNNING_PIPELINE state') _monitorPipeline(batchState) yield _waitForPipeline(batchState) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed running pipeline' ).progress()) batchState['pipeline_state'] = POSTRUN_STATE state.updateBatchState() if batchState['pipeline_state'] == POSTRUN_STATE: _log(batchState, 'Pipeline is in POSTRUN state') yield state.postrunQueue.addWithDeferred(workflow_runner.run, state.workflowConfig(), batchState['pipeline_config']['pipeline.POSTRUN_TEMPLATE_XML'], pipelineConfigFile, TMP_DIR) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed postrun' ).progress()) batchState['pipeline_state'] = COMPLETED_STATE batchState['state'] = COMPLETED_STATE state.updateBatchState() yield _updateTask(batchState, lambda t : t.setState(tasks.task.TASK_COMPLETED)) _log(batchState, 'Pipeline finished successfully')
def handleWWWRunPipeline(request): """ In the case of a pipeline we will do all the work necessary to run the pipeline and then setup a listener to run in the background tracking its progress. If bare_run is False then the pipeline run will actually be wrapped in `clovr_wrapper`. Otherwise a pipeline of the type pipeline.PIPELINE_TEMPLATE is run. Input: { cluster: string user_name: string ?parent_pipeline: string ?queue: string ?overwrite: boolean bare_run: boolean config: { key/value } } Output: lite_pipeline """ @defer.inlineCallbacks def _createPipeline(request): taskName = yield tasks_tx.createTaskAndSave('runPipelines', 0) # The name of a pipeline is being stored as a checksum. Pipeline names # are arbitrary and the user will likely never know or care what it is. # The pipeline name still exists though because other tools will likely # find it useful to refer to a pipeline by a particular name, but if # we decide to change the pipeline name to something more meaningful they # won't have to chagne their code to use pipelineName instead of checksum protocol = _determineProtocol(request) if not request.body['bare_run']: request.body['config'][ 'pipeline.PIPELINE_WRAPPER_NAME'] = request.body['config'][ 'pipeline.PIPELINE_NAME'] defer.returnValue( persist.Pipeline(pipelineId=None, pipelineName=checksum, userName=request.body['user_name'], protocol=protocol, checksum=checksum, taskName=taskName, queue=request.body.get('queue', 'pipelinewrapper.q'), children=[], config=request.body['config'])) @defer.inlineCallbacks def _startRemotePipeline(request): clusters = yield clusters_client.listClusters( {'cluster_name': request.body['cluster']}, request.body['user_name']) cluster = clusters[0] # Forward the request on to the remote cluster, set parent_pipeline to None ret = yield pipelines_www_client.runPipeline( cluster['master']['public_dns'], 'local', request.body['user_name'], None, request.body['bare_run'], request.body.get('queue', 'pipelinewrapper.q'), request.body['config'], request.body.get('overwrite', False)) defer.returnValue(ret) # If the parent pipeline is set and doesn't exist, error if request.body.get('parent_pipeline'): parentPipelines = yield request.state.pipelinePersist.loadAllPipelinesBy( {'pipeline_name': request.body['parent_pipeline']}, request.body['user_name']) if not parentPipelines: raise InvalidParentPipeline(request.body['parent_pipeline']) if len(parentPipelines) == 1: parentPipeline = parentPipelines[0] else: raise Exception( 'More than one possible parent pipeline choice, not sure what to do here' ) else: parentPipeline = None if request.body['cluster'] == 'local': checksum = pipeline_misc.checksumInput(request.body['config']) protocol = _determineProtocol(request) if protocol == 'clovr_batch_wrapper': errors = yield pipeline_misc.validateBatchPipelineConfig(request) else: errors = yield pipeline_misc.validatePipelineConfig(request) if errors: raise InvalidPipelineConfig( 'Configuration did not pass validation') request.body['config']['pipeline.PIPELINE_NAME'] = checksum try: # Pretty lame way to force control to the exceptional case # We aren't in a try block just for this line, though. The line # that loads the pipeline could also fail if request.body.get('overwrite', False): raise persist.PipelineNotFoundError('flow control') existingPipeline = yield request.state.pipelinePersist.loadPipelineBy( { 'checksum': checksum, 'protocol': protocol }, request.body['user_name']) pipelineDict = yield request.state.pipelinesCache.pipelineToDict( existingPipeline) defer.returnValue(request.update(response=pipelineDict)) except persist.PipelineNotFoundError: pipeline = yield _createPipeline(request) yield request.state.pipelinePersist.savePipeline(pipeline) # We want to do a deeper validation of the configuration and then run the pipeline. # Then we want to monitor it both through the ergatis observer and a timed update # of any children it has. # # We are going to do all this work in the background so we can exit the # handler. Since incoming requests are rate-limited, we don't want to # block the handler for too long. In this case we weren't pushing the # request and pipleine onto the queue for another handler to pick up # like we do in many other cases because we don't have to. Deeper # validation is through a tasklet which is rate limited and submitting # a pipeline and monitoring it are all fairly light operations. d = pipeline_misc.deepValidation(request, pipeline) d.addCallback(lambda p: pipeline_misc.runPipeline(request, p)) # runPipeline returns a pipeline monitor, not a pipeline d.addCallback( lambda pm: request.state.pipelinePersist.savePipeline( pm.pipeline).addCallback(lambda _: pm.pipeline)) d.addErrback(lambda f: tasks_tx.updateTask( pipeline.taskName, lambda t: t.setState( tasks_tx.task.TASK_FAILED).addFailure(f))) pipelineDict = yield request.state.pipelinesCache.pipelineToDict( pipeline) if parentPipeline: parentPipeline = parentPipeline.update(children=list( set([ tuple(e) for e in parentPipeline.children + [('local', pipeline.pipelineName)] ]))) yield request.state.pipelinePersist.savePipeline( parentPipeline) defer.returnValue(request.update(response=pipelineDict)) else: pipelineDict = yield _startRemotePipeline(request) if parentPipeline: childPipeline = [(request.body['cluster'], pipelineDict['pipeline_name'])] parentPipeline = parentPipeline.update(children=list( set([ tuple(e) for e in parentPipeline.children + childPipeline ]))) yield request.state.pipelinePersist.savePipeline(parentPipeline) defer.returnValue(request.update(response=pipelineDict))
def handleWWWRunPipeline(request): """ In the case of a pipeline we will do all the work necessary to run the pipeline and then setup a listener to run in the background tracking its progress. If bare_run is False then the pipeline run will actually be wrapped in `clovr_wrapper`. Otherwise a pipeline of the type pipeline.PIPELINE_TEMPLATE is run. Input: { cluster: string user_name: string ?parent_pipeline: string ?queue: string ?overwrite: boolean bare_run: boolean config: { key/value } } Output: lite_pipeline """ @defer.inlineCallbacks def _createPipeline(request): taskName = yield tasks_tx.createTaskAndSave('runPipelines', 0) # The name of a pipeline is being stored as a checksum. Pipeline names # are arbitrary and the user will likely never know or care what it is. # The pipeline name still exists though because other tools will likely # find it useful to refer to a pipeline by a particular name, but if # we decide to change the pipeline name to something more meaningful they # won't have to chagne their code to use pipelineName instead of checksum protocol = _determineProtocol(request) if not request.body['bare_run']: request.body['config']['pipeline.PIPELINE_WRAPPER_NAME'] = request.body['config']['pipeline.PIPELINE_NAME'] defer.returnValue(persist.Pipeline(pipelineId=None, pipelineName=checksum, userName=request.body['user_name'], protocol=protocol, checksum=checksum, taskName=taskName, queue=request.body.get('queue', 'pipelinewrapper.q'), children=[], config=request.body['config'])) @defer.inlineCallbacks def _startRemotePipeline(request): clusters = yield clusters_client.listClusters({'cluster_name': request.body['cluster']}, request.body['user_name']) cluster = clusters[0] # Forward the request on to the remote cluster, set parent_pipeline to None ret = yield pipelines_www_client.runPipeline(cluster['master']['public_dns'], 'local', request.body['user_name'], None, request.body['bare_run'], request.body.get('queue', 'pipelinewrapper.q'), request.body['config'], request.body.get('overwrite', False)) defer.returnValue(ret) # If the parent pipeline is set and doesn't exist, error if request.body.get('parent_pipeline'): parentPipelines = yield request.state.pipelinePersist.loadAllPipelinesBy({'pipeline_name': request.body['parent_pipeline']}, request.body['user_name']) if not parentPipelines: raise InvalidParentPipeline(request.body['parent_pipeline']) if len(parentPipelines) == 1: parentPipeline = parentPipelines[0] else: raise Exception('More than one possible parent pipeline choice, not sure what to do here') else: parentPipeline = None if request.body['cluster'] == 'local': checksum = pipeline_misc.checksumInput(request.body['config']) protocol = _determineProtocol(request) if protocol == 'clovr_batch_wrapper': errors = yield pipeline_misc.validateBatchPipelineConfig(request) else: errors = yield pipeline_misc.validatePipelineConfig(request) if errors: raise InvalidPipelineConfig('Configuration did not pass validation') request.body['config']['pipeline.PIPELINE_NAME'] = checksum try: # Pretty lame way to force control to the exceptional case # We aren't in a try block just for this line, though. The line # that loads the pipeline could also fail if request.body.get('overwrite', False): raise persist.PipelineNotFoundError('flow control') existingPipeline = yield request.state.pipelinePersist.loadPipelineBy({'checksum': checksum, 'protocol': protocol}, request.body['user_name']) pipelineDict = yield request.state.pipelinesCache.pipelineToDict(existingPipeline) defer.returnValue(request.update(response=pipelineDict)) except persist.PipelineNotFoundError: pipeline = yield _createPipeline(request) yield request.state.pipelinePersist.savePipeline(pipeline) # We want to do a deeper validation of the configuration and then run the pipeline. # Then we want to monitor it both through the ergatis observer and a timed update # of any children it has. # # We are going to do all this work in the background so we can exit the # handler. Since incoming requests are rate-limited, we don't want to # block the handler for too long. In this case we weren't pushing the # request and pipleine onto the queue for another handler to pick up # like we do in many other cases because we don't have to. Deeper # validation is through a tasklet which is rate limited and submitting # a pipeline and monitoring it are all fairly light operations. d = pipeline_misc.deepValidation(request, pipeline) d.addCallback(lambda p : pipeline_misc.runPipeline(request, p)) # runPipeline returns a pipeline monitor, not a pipeline d.addCallback(lambda pm : request.state.pipelinePersist.savePipeline(pm.pipeline).addCallback(lambda _ : pm.pipeline)) d.addErrback(lambda f : tasks_tx.updateTask(pipeline.taskName, lambda t : t.setState(tasks_tx.task.TASK_FAILED).addFailure(f))) pipelineDict = yield request.state.pipelinesCache.pipelineToDict(pipeline) if parentPipeline: parentPipeline = parentPipeline.update(children=list(set([tuple(e) for e in parentPipeline.children + [('local', pipeline.pipelineName)]]))) yield request.state.pipelinePersist.savePipeline(parentPipeline) defer.returnValue(request.update(response=pipelineDict)) else: pipelineDict = yield _startRemotePipeline(request) if parentPipeline: childPipeline = [(request.body['cluster'], pipelineDict['pipeline_name'])] parentPipeline = parentPipeline.update(children=list(set([tuple(e) for e in parentPipeline.children + childPipeline]))) yield request.state.pipelinePersist.savePipeline(parentPipeline) defer.returnValue(request.update(response=pipelineDict))
def _run(state, batchState): if 'pipeline_name' in batchState: pipelines = yield pipelines_client.pipelineList( 'localhost', 'local', 'guest', batchState['pipeline_name'], detail=True) else: pipelines = [] if not pipelines: _log(batchState, 'First time running, creating pipeline state information') batchState['pipeline_config'] = yield _applyActions( state.innerPipelineConfig(), batchState['actions']) batchState['pipeline_state'] = STARTCLUSTER_STATE # We need to create a fake, local, pipeline for metrics to work batchState['pipeline_name'] = pipeline_misc.checksumInput( batchState['pipeline_config']) batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState[ 'pipeline_name'] batchState['pipeline_config'][ 'pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name'] _log(batchState, 'Pipeline named ' + batchState['pipeline_name']) pipeline = yield pipelines_client.createPipeline( host='localhost', clusterName='local', userName='******', pipelineName=batchState['pipeline_name'], protocol='clovr_wrapper', queue='pipeline.q', config=batchState['pipeline_config'], parentPipeline=state.parentPipeline()) batchState['lgt_wrapper_task_name'] = pipeline['task_name'] _log( batchState, 'Setting number of tasks to 9 (number in a standard lgt_wrapper)') yield _updateTask(batchState, lambda t: t.update(completedTasks=0, numTasks=9)) state.updateBatchState() batchState['state'] = RUNNING_STATE _log(batchState, 'Pipeline started in %s state' % batchState['pipeline_state']) yield _updateTask(batchState, lambda t: t.setState(tasks.task.TASK_RUNNING)) pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs', global_state.make_ref() + '.conf') _log(batchState, 'Creating ergatis configuration') _writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile) if batchState['pipeline_state'] == STARTCLUSTER_STATE: _log(batchState, 'Pipeline is in STARTCLUSTER state') # First see if the cluster exists but is unresponsive try: cluster = yield loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, terminating') terminateTask = yield clusters_client.terminateCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _blockOnTask(terminateTask) except: pass batchState['cluster_task'] = yield startCluster( batchState, 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', int(batchState['pipeline_config']['cluster.EXEC_NODES']), 0, batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'], { 'cluster.MASTER_INSTANCE_TYPE': batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'], 'cluster.MASTER_BID_PRICE': batchState['pipeline_config']['cluster.MASTER_BID_PRICE'], 'cluster.EXEC_INSTANCE_TYPE': batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'], 'cluster.EXEC_BID_PRICE': batchState['pipeline_config']['cluster.EXEC_BID_PRICE'] }) yield _updateTask(batchState, lambda t: t.update(completedTasks=0, numTasks=9)) yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed startcluster').progress()) batchState['pipeline_state'] = REMOTE_LOCAL_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REMOTE_LOCAL_TRANSFER_STATE: _log(batchState, 'Pipeline is in REMOTE_LOCAL_TRANSFER') _log(batchState, 'Making sure cluster is exists in some form') cluster = yield clusters_client.loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, erroring and restarting') raise Exception('Cluster is not responsive') yield state.prerunQueue.addWithDeferred(_remoteLocalTransfer, batchState) yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed remote_local_transfer'). progress()) batchState['pipeline_state'] = DECRYPT_STATE state.updateBatchState() if batchState['pipeline_state'] == DECRYPT_STATE: _log(batchState, 'Pipeline is in DECRYPT') cluster = yield loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') tag = yield tags_client.loadTag( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', _decryptTagName(batchState)) conf = config.configFromStream(open('/tmp/machine.conf')) yield ssh.runProcessSSH(cluster['master']['public_dns'], 'mkdir -p /mnt/lgt_decrypt', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) yield rsync.rsyncTo( cluster['master']['public_dns'], batchState['pipeline_config']['params.DECRYPT_SCRIPT'], '/mnt/', options=conf('rsync.options'), user=conf('rsync.user'), log=True) for f in tag['files']: decryptCmd = ' '.join([ os.path.join( '/mnt', os.path.basename(batchState['pipeline_config'] ['params.DECRYPT_SCRIPT'])), f, '-out-dir', '/mnt/lgt_decrypt', '-remove-encrypted', '-password', batchState['pipeline_config']['params.DECRYPT_PASSWORD'] ]) yield ssh.getOutput(cluster['master']['public_dns'], decryptCmd, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), expected=[0, 253], log=True) tag = yield tags_client.tagData( host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', action='overwrite', tagName=_decryptTagName(batchState), files=['/mnt/lgt_decrypt'], metadata={}, recursive=True, expand=False, compressDir=None) _log( batchState, 'Waiting for tagging of %s to complete - %s' % (_decryptTagName(batchState), tag['task_name'])) yield _blockOnTask( tag['task_name'], cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME']) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed decrypt').progress()) batchState['pipeline_state'] = REFERENCE_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REFERENCE_TRANSFER_STATE: _log(batchState, 'Pipeline is in REFERENCE_TRANSFER state') transfers = [] tags = (batchState['pipeline_config']['input.REF_TAG1'].split(',') + batchState['pipeline_config']['input.REF_TAG2'].split(',')) for tag in tags: tag = tag.strip() output = yield _getOutput(batchState, [ 'vp-transfer-dataset', '-t', '--tag-name=' + tag, '--dst-cluster=' + batchState['pipeline_config']['cluster.CLUSTER_NAME'] ], log=True) transfers.append(output['stdout'].strip()) for task in transfers: yield _blockOnTask(task) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed reference_transfer').progress()) batchState['pipeline_state'] = RUN_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUN_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUN_PIPELINE state') batchState['pipeline_config']['input.INPUT_TAG'] = _decryptTagName( batchState) pipeline = yield pipelines_client.runPipeline( host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', parentPipeline=batchState['pipeline_name'], bareRun=True, queue=state.innerPipelineQueue(), config=batchState['pipeline_config'], overwrite=True) batchState['pipeline_task'] = pipeline['task_name'] yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed run pipeline').progress()) batchState['pipeline_state'] = RUNNING_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUNNING_PIPELINE state') _monitorPipeline(batchState) yield _waitForPipeline(batchState) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed running pipeline').progress()) batchState['pipeline_state'] = HARVEST_STATE state.updateBatchState() if batchState['pipeline_state'] == HARVEST_STATE: _log(batchState, 'Pipeline is in HARVEST state') # Using prerunqueue because we want everything here serialized yield state.prerunQueue.addWithDeferred(_harvestTransfer, batchState) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed harvest').progress()) batchState['pipeline_state'] = SHUTDOWN_STATE state.updateBatchState() if batchState['pipeline_state'] == SHUTDOWN_STATE: _log(batchState, 'Pipeline is in SHUTDOWN state') if 'add_instances_task' in batchState: try: yield _blockOnTask(batchState['add_instances_task'], cluster=batchState['pipeline_config'] ['cluster.CLUSTER_NAME']) except Exception, err: logging.errorPrint(str(err)) log.err(err) yield clusters_client.terminateCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed shutdown').progress()) batchState['pipeline_state'] = COMPLETED_STATE batchState['state'] = COMPLETED_STATE state.updateBatchState()