def _createDir(d): yield ssh.runProcessSSH(cluster['master']['public_dns'], 'mkdir -p ' + d, None, log.err, cluster['config']['ssh.user'], cluster['config']['ssh.options']) try: if cluster['master']['instance_type'] is not None: yield ssh.runProcessSSH( cluster['master']['public_dns'], 'chown -R %s %s' % (cluster['config']['vappio.user'], d), None, log.err, cluster['config']['ssh.user'], cluster['config']['ssh.options']) except commands.Error: pass
def _setQueue(taskName, batchState): yield _blockOnTask(taskName) cluster = yield loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield defer_utils.tryUntil( 10, lambda: _getOutput(batchState, [ '/opt/clovr_pipelines/workflow/project_saved_templates/clovr_lgt_wrapper/set_queue.sh', cluster['master']['public_dns'] ], log=True), onFailure=defer_utils.sleep(2)) conf = config.configFromStream(open('/tmp/machine.conf')) # Remove autoshutdown, we want no part of that yield ssh.runProcessSSH(cluster['master']['public_dns'], 'rm -v /var/vappio/runtime/noautoshutdown', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True)
def _delayAutoshutdown(state, batchState): _log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file') try: clusters = yield clusters_client.listClusters('localhost', {'cluster_name': batchState['pipeline_config']['cluster.CLUSTER_NAME']}, 'guest') cluster = clusters[0] if batchState.get('state', None) == COMPLETED_STATE: _log(batchState, 'AUTOSHUTDOWN: Pipeline complete, done') if batchState.get('state', None) != RUNNING_STATE: _log(batchState, 'AUTOSHUTDOWN: Pipeline not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) elif cluster['state'] == 'running': # We need the SSH options from the machine.conf, ugh I hate these OOB dependencies conf = config.configFromStream(open('/tmp/machine.conf')) _log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown') yield ssh.runProcessSSH(cluster['master']['public_dns'], 'touch /var/vappio/runtime/delayautoshutdown', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) _log(batchState, 'AUTOSHUTDOWN: Setting up next call') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) else: _log(batchState, 'AUTOSHUTDOWN: Cluster not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) except: _log(batchState, 'AUTOSHUTDOWN: Cluster does not exist, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
def _waitForSSH(instance): try: yield ssh.runProcessSSH(instance['public_dns'], 'echo hello', None, None, sshUser, sshOptions) except: defer.returnValue(False) defer.returnValue(True)
def _createDir(d): yield ssh.runProcessSSH(cluster['master']['public_dns'], 'mkdir -p ' + d, None, log.err, cluster['config']['ssh.user'], cluster['config']['ssh.options']) try: if cluster['master']['instance_type'] is not None: yield ssh.runProcessSSH(cluster['master']['public_dns'], 'chown -R %s %s' % (cluster['config']['vappio.user'], d), None, log.err, cluster['config']['ssh.user'], cluster['config']['ssh.options']) except commands.Error: pass
def _waitForBoot(instance): try: yield ssh.runProcessSSH(instance['public_dns'], 'test -e %s' % fileName, None, None, sshUser, sshOptions) except: ## Should differentiate between SSH failing and file not ## existings so we can return FAILED_INSTANCE appropriately defer.returnValue(False) defer.returnValue(True)
def _sshAble(instance): sshUser = state.clustersState.machineConf('ssh.user') sshOptions = state.clustersState.machineConf('ssh.options') try: yield ssh.runProcessSSH(instance['public_dns'], 'echo hello', None, None, sshUser, sshOptions) except: defer.returnValue(False) defer.returnValue(True)
def loadRemoteCluster(state, cl): """ Tries to load a cluster. Returns the cluster on success otherwise throws an error. If the cluster is not actually owned by us throws auth_token.AuthTokenError If it's unresponsive throws errors.RemoteError We also check for SSH being up and throw a RemoteError if it is not responsive """ if cl.master: authToken = auth_token.generateToken( cl.config('cluster.cluster_public_key')) try: clusters = yield clusters_www_client.listClusters( cl.master['public_dns'], {'cluster_name': 'local'}, None, authToken, timeout=10, tries=3) cluster = clusters[0] yield ssh.runProcessSSH(cl.master['public_dns'], 'echo hello', stdoutf=None, stderrf=None, sshUser=state.machineConf('ssh.user'), sshFlags=state.machineConf('ssh.options')) defer.returnValue(cluster) except errors.RemoteError, err: if err.name == 'igs.utils.auth_token.AuthTokenError': raise auth_token.AuthTokenError() else: raise except commands.ProgramRunError: raise errors.RemoteError('SSH failed')
def _delayAutoshutdown(state, batchState): _log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file') try: clusters = yield clusters_client.listClusters('localhost', { 'cluster_name': batchState['pipeline_config']['cluster.CLUSTER_NAME'] }, 'guest') cluster = clusters[0] if batchState.get('state', None) == COMPLETED_STATE: _log(batchState, 'AUTOSHUTDOWN: Pipeline complete, done') if batchState.get('state', None) != RUNNING_STATE: _log(batchState, 'AUTOSHUTDOWN: Pipeline not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) elif cluster['state'] == 'running': # We need the SSH options from the machine.conf, ugh I hate these OOB dependencies conf = config.configFromStream(open('/tmp/machine.conf')) _log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown') yield ssh.runProcessSSH( cluster['master']['public_dns'], 'touch /var/vappio/runtime/delayautoshutdown', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) _log(batchState, 'AUTOSHUTDOWN: Setting up next call') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) else: _log(batchState, 'AUTOSHUTDOWN: Cluster not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) except: _log(batchState, 'AUTOSHUTDOWN: Cluster does not exist, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState)
def loadRemoteCluster(state, cl): """ Tries to load a cluster. Returns the cluster on success otherwise throws an error. If the cluster is not actually owned by us throws auth_token.AuthTokenError If it's unresponsive throws errors.RemoteError We also check for SSH being up and throw a RemoteError if it is not responsive """ if cl.master: authToken = auth_token.generateToken(cl.config('cluster.cluster_public_key')) try: clusters = yield clusters_www_client.listClusters(cl.master['public_dns'], {'cluster_name': 'local'}, None, authToken, timeout=10, tries=3) cluster = clusters[0] yield ssh.runProcessSSH(cl.master['public_dns'], 'echo hello', stdoutf=None, stderrf=None, sshUser=state.machineConf('ssh.user'), sshFlags=state.machineConf('ssh.options')) defer.returnValue(cluster) except errors.RemoteError, err: if err.name == 'igs.utils.auth_token.AuthTokenError': raise auth_token.AuthTokenError() else: raise except commands.ProgramRunError: raise errors.RemoteError('SSH failed')
def _delayAutoshutdown(state, batchState): _log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file') if (('cluster_task' not in batchState or batchState.get('state', None) != RUNNING_STATE) and batchState.get('state', None) != COMPLETED_STATE): # Not ready yet _log(batchState, 'AUTOSHUTDOWN: No cluster or not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) elif ('cluster_task' in batchState and batchState.get('state', None) == RUNNING_STATE and batchState.get('pipeline_state', None) != SHUTDOWN_STATE): # Ready to see if resizing _log(batchState, 'AUTOSHUTDOWN: Making sure cluster is up') yield _blockOnTask(batchState['cluster_task']) try: cluster = yield loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') # We need the SSH options from the machine.conf, ugh I hate these OOB dependencies conf = config.configFromStream(open('/tmp/machine.conf')) _log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown') yield ssh.runProcessSSH( cluster['master']['public_dns'], 'touch /var/vappio/runtime/delayautoshutdown', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) except: pass _log(batchState, 'AUTOSHUTDOWN: Setting up next call') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) else: _log(batchState, 'AUTOSHUTDOWN: Giving up on this')
def _delayAutoshutdown(state, batchState): _log(batchState, 'AUTOSHUTDOWN: Trying to touch autoshutdown file') if (('cluster_task' not in batchState or batchState.get('state', None) != RUNNING_STATE) and batchState.get('state', None) != COMPLETED_STATE): # Not ready yet _log(batchState, 'AUTOSHUTDOWN: No cluster or not running, calling later') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) elif ('cluster_task' in batchState and batchState.get('state', None) == RUNNING_STATE and batchState.get('pipeline_state', None) != SHUTDOWN_STATE): # Ready to see if resizing _log(batchState, 'AUTOSHUTDOWN: Making sure cluster is up') yield _blockOnTask(batchState['cluster_task']) try: cluster = yield loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') # We need the SSH options from the machine.conf, ugh I hate these OOB dependencies conf = config.configFromStream(open('/tmp/machine.conf')) _log(batchState, 'AUTOSHUTDOWN: Touching delayautoshutdown') yield ssh.runProcessSSH(cluster['master']['public_dns'], 'touch /var/vappio/runtime/delayautoshutdown', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) except: pass _log(batchState, 'AUTOSHUTDOWN: Setting up next call') reactor.callLater(AUTOSHUTDOWN_REFRESH, _delayAutoshutdown, state, batchState) else: _log(batchState, 'AUTOSHUTDOWN: Giving up on this')
def _setQueue(taskName, batchState): yield _blockOnTask(taskName) cluster = yield loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield defer_utils.tryUntil(10, lambda : _getOutput(batchState, ['/opt/clovr_pipelines/workflow/project_saved_templates/clovr_lgt_wrapper/set_queue.sh', cluster['master']['public_dns']], log=True), onFailure=defer_utils.sleep(2)) conf = config.configFromStream(open('/tmp/machine.conf')) # Remove autoshutdown, we want no part of that yield ssh.runProcessSSH(cluster['master']['public_dns'], 'rm -v /var/vappio/runtime/noautoshutdown', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True)
def _uploadTag(request): localTag = yield request.state.tagPersist.loadTag(request.body['tag_name']) srcClusters = yield www_clusters.listClusters('localhost', {'cluster_name': request.body['src_cluster']}, request.body['user_name']) srcCluster = srcClusters[0] dstClusters = yield www_clusters.listClusters('localhost', {'cluster_name': request.body['dst_cluster']}, request.body['user_name']) dstCluster = dstClusters[0] # We want the trailing '/' so everyone knows it's a directory dstTagPath = os.path.join(dstCluster['config']['dirs.upload_dir'], localTag.tagName) + '/' # Change dstTagPath to use /mnt/nostaging instead of mnt/staging as the destination if the # tag has a nostaging metadata value #if localTag.metadata['nostaging']: # dstTagPath = os.path.join(dstCluster['config']['dirs.nostaging_upload_dir'], localTag.tagName) + '/' if request.body['dst_type'] == 'local' : upload_dir = '/mnt/nostaging/' if 'dirs.nostaging_upload_dir' in dstCluster['config']: upload_dir = dstCluster['config']['dirs.nostaging_upload_dir'] yield _makeDirsOnCluster(dstCluster, [upload_dir]) dstTagPath = os.path.join(upload_dir, localTag.tagName) + '/' baseDirFiles, nonBaseDirFiles = _partitionFiles(localTag.files, localTag.metadata['tag_base_dir']) if baseDirFiles: yield rsync.rsyncTo(dstCluster['master']['public_dns'], localTag.metadata['tag_base_dir'], dstTagPath, baseDirFiles, srcCluster['config']['rsync.options'], srcCluster['config']['rsync.user'], log=True) if nonBaseDirFiles: yield rsync.rsyncTo(dstCluster['master']['public_dns'], '/', dstTagPath, nonBaseDirFiles, srcCluster['config']['rsync.options'], srcCluster['config']['rsync.user'], log=True) remoteFiles = ([os.path.join(dstTagPath, f) for f in baseDirFiles] + [os.path.join(dstTagPath, _makePathRelative(f)) for f in nonBaseDirFiles]) metadata = localTag.metadata if metadata.get('urls', []) and not metadata.get('urls_realized', False): tag = yield _realizeUrls(request) remoteFiles.extend(tag['files']) metadata = func.updateDict(metadata, {'urls_realized': True}) # If we are dealing with an adhoc cluster here there is a chance that # we are attempting to transfer to a local VM's shared folder which does # support chown/chmod so we will skip this step if dstCluster['master']['instance_type'] is not None: yield ssh.runProcessSSH(dstCluster['master']['public_dns'], 'chown -R %s %s' % (dstCluster['config']['vappio.user'], dstTagPath), None, log.err, srcCluster['config']['ssh.user'], srcCluster['config']['ssh.options']) defer.returnValue(persist.Tag(tagName=localTag.tagName, files=remoteFiles, metadata=func.updateDict(metadata, {'tag_base_dir': dstTagPath}), phantom=localTag.phantom, taskName=None))
def _run(state, batchState): if 'pipeline_name' in batchState: pipelines = yield pipelines_client.pipelineList('localhost', 'local', 'guest', batchState['pipeline_name'], detail=True) else: pipelines = [] if not pipelines: _log(batchState, 'First time running, creating pipeline state information') batchState['pipeline_config'] = yield _applyActions(state.innerPipelineConfig(), batchState['actions']) batchState['pipeline_state'] = STARTCLUSTER_STATE # We need to create a fake, local, pipeline for metrics to work batchState['pipeline_name'] = pipeline_misc.checksumInput(batchState['pipeline_config']) batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState['pipeline_name'] batchState['pipeline_config']['pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name'] _log(batchState, 'Pipeline named ' + batchState['pipeline_name']) pipeline = yield pipelines_client.createPipeline(host='localhost', clusterName='local', userName='******', pipelineName=batchState['pipeline_name'], protocol='clovr_wrapper', queue='pipeline.q', config=batchState['pipeline_config'], parentPipeline=state.parentPipeline()) batchState['lgt_wrapper_task_name'] = pipeline['task_name'] _log(batchState, 'Setting number of tasks to 9 (number in a standard lgt_wrapper)') yield _updateTask(batchState, lambda t : t.update(completedTasks=0, numTasks=9)) state.updateBatchState() batchState['state'] = RUNNING_STATE _log(batchState, 'Pipeline started in %s state' % batchState['pipeline_state']) yield _updateTask(batchState, lambda t : t.setState(tasks.task.TASK_RUNNING)) pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs', global_state.make_ref() + '.conf') _log(batchState, 'Creating ergatis configuration') _writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile) if batchState['pipeline_state'] == STARTCLUSTER_STATE: _log(batchState, 'Pipeline is in STARTCLUSTER state') # First see if the cluster exists but is unresponsive try: cluster = yield loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, terminating') terminateTask = yield clusters_client.terminateCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _blockOnTask(terminateTask) except: pass batchState['cluster_task'] = yield startCluster( batchState, 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', int(batchState['pipeline_config']['cluster.EXEC_NODES']), 0, batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'], {'cluster.MASTER_INSTANCE_TYPE': batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'], 'cluster.MASTER_BID_PRICE': batchState['pipeline_config']['cluster.MASTER_BID_PRICE'], 'cluster.EXEC_INSTANCE_TYPE': batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'], 'cluster.EXEC_BID_PRICE': batchState['pipeline_config']['cluster.EXEC_BID_PRICE']}) yield _updateTask(batchState, lambda t : t.update(completedTasks=0, numTasks=9)) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed startcluster' ).progress()) batchState['pipeline_state'] = REMOTE_LOCAL_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REMOTE_LOCAL_TRANSFER_STATE: _log(batchState, 'Pipeline is in REMOTE_LOCAL_TRANSFER') _log(batchState, 'Making sure cluster is exists in some form') cluster = yield clusters_client.loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, erroring and restarting') raise Exception('Cluster is not responsive') yield state.prerunQueue.addWithDeferred(_remoteLocalTransfer, batchState) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed remote_local_transfer' ).progress()) batchState['pipeline_state'] = DECRYPT_STATE state.updateBatchState() if batchState['pipeline_state'] == DECRYPT_STATE: _log(batchState, 'Pipeline is in DECRYPT') cluster = yield loadCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') tag = yield tags_client.loadTag('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', _decryptTagName(batchState)) conf = config.configFromStream(open('/tmp/machine.conf')) yield ssh.runProcessSSH(cluster['master']['public_dns'], 'mkdir -p /mnt/lgt_decrypt', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) yield rsync.rsyncTo(cluster['master']['public_dns'], batchState['pipeline_config']['params.DECRYPT_SCRIPT'], '/mnt/', options=conf('rsync.options'), user=conf('rsync.user'), log=True) for f in tag['files']: decryptCmd = ' '.join([os.path.join('/mnt', os.path.basename(batchState['pipeline_config']['params.DECRYPT_SCRIPT'])), f, '-out-dir', '/mnt/lgt_decrypt', '-remove-encrypted', '-password', batchState['pipeline_config']['params.DECRYPT_PASSWORD']]) yield ssh.getOutput(cluster['master']['public_dns'], decryptCmd, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), expected=[0, 253], log=True) tag = yield tags_client.tagData(host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', action='overwrite', tagName=_decryptTagName(batchState), files=['/mnt/lgt_decrypt'], metadata={}, recursive=True, expand=False, compressDir=None) _log(batchState, 'Waiting for tagging of %s to complete - %s' % (_decryptTagName(batchState), tag['task_name'])) yield _blockOnTask(tag['task_name'], cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME']) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed decrypt' ).progress()) batchState['pipeline_state'] = REFERENCE_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REFERENCE_TRANSFER_STATE: _log(batchState, 'Pipeline is in REFERENCE_TRANSFER state') transfers = [] tags = (batchState['pipeline_config']['input.REF_TAG1'].split(',') + batchState['pipeline_config']['input.REF_TAG2'].split(',')) for tag in tags: tag = tag.strip() output = yield _getOutput(batchState, ['vp-transfer-dataset', '-t', '--tag-name=' + tag, '--dst-cluster=' + batchState['pipeline_config']['cluster.CLUSTER_NAME']], log=True) transfers.append(output['stdout'].strip()) for task in transfers: yield _blockOnTask(task) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed reference_transfer' ).progress()) batchState['pipeline_state'] = RUN_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUN_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUN_PIPELINE state') batchState['pipeline_config']['input.INPUT_TAG'] = _decryptTagName(batchState) pipeline = yield pipelines_client.runPipeline(host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', parentPipeline=batchState['pipeline_name'], bareRun=True, queue=state.innerPipelineQueue(), config=batchState['pipeline_config'], overwrite=True) batchState['pipeline_task'] = pipeline['task_name'] yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed run pipeline' ).progress()) batchState['pipeline_state'] = RUNNING_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUNNING_PIPELINE state') _monitorPipeline(batchState) yield _waitForPipeline(batchState) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed running pipeline' ).progress()) batchState['pipeline_state'] = HARVEST_STATE state.updateBatchState() if batchState['pipeline_state'] == HARVEST_STATE: _log(batchState, 'Pipeline is in HARVEST state') # Using prerunqueue because we want everything here serialized yield state.prerunQueue.addWithDeferred(_harvestTransfer, batchState) yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed harvest' ).progress()) batchState['pipeline_state'] = SHUTDOWN_STATE state.updateBatchState() if batchState['pipeline_state'] == SHUTDOWN_STATE: _log(batchState, 'Pipeline is in SHUTDOWN state') if 'add_instances_task' in batchState: try: yield _blockOnTask(batchState['add_instances_task'], cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME']) except Exception, err: logging.errorPrint(str(err)) log.err(err) yield clusters_client.terminateCluster('localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _updateTask(batchState, lambda t : t.addMessage(tasks.task.MSG_SILENT, 'Completed shutdown' ).progress()) batchState['pipeline_state'] = COMPLETED_STATE batchState['state'] = COMPLETED_STATE state.updateBatchState()
def _run(state, batchState): if 'pipeline_name' in batchState: pipelines = yield pipelines_client.pipelineList( 'localhost', 'local', 'guest', batchState['pipeline_name'], detail=True) else: pipelines = [] if not pipelines: _log(batchState, 'First time running, creating pipeline state information') batchState['pipeline_config'] = yield _applyActions( state.innerPipelineConfig(), batchState['actions']) batchState['pipeline_state'] = STARTCLUSTER_STATE # We need to create a fake, local, pipeline for metrics to work batchState['pipeline_name'] = pipeline_misc.checksumInput( batchState['pipeline_config']) batchState['pipeline_config']['pipeline.PIPELINE_NAME'] = batchState[ 'pipeline_name'] batchState['pipeline_config'][ 'pipeline.PIPELINE_WRAPPER_NAME'] = batchState['pipeline_name'] _log(batchState, 'Pipeline named ' + batchState['pipeline_name']) pipeline = yield pipelines_client.createPipeline( host='localhost', clusterName='local', userName='******', pipelineName=batchState['pipeline_name'], protocol='clovr_wrapper', queue='pipeline.q', config=batchState['pipeline_config'], parentPipeline=state.parentPipeline()) batchState['lgt_wrapper_task_name'] = pipeline['task_name'] _log( batchState, 'Setting number of tasks to 9 (number in a standard lgt_wrapper)') yield _updateTask(batchState, lambda t: t.update(completedTasks=0, numTasks=9)) state.updateBatchState() batchState['state'] = RUNNING_STATE _log(batchState, 'Pipeline started in %s state' % batchState['pipeline_state']) yield _updateTask(batchState, lambda t: t.setState(tasks.task.TASK_RUNNING)) pipelineConfigFile = os.path.join(TMP_DIR, 'pipeline_configs', global_state.make_ref() + '.conf') _log(batchState, 'Creating ergatis configuration') _writeErgatisConfig(batchState['pipeline_config'], pipelineConfigFile) if batchState['pipeline_state'] == STARTCLUSTER_STATE: _log(batchState, 'Pipeline is in STARTCLUSTER state') # First see if the cluster exists but is unresponsive try: cluster = yield loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, terminating') terminateTask = yield clusters_client.terminateCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _blockOnTask(terminateTask) except: pass batchState['cluster_task'] = yield startCluster( batchState, 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', int(batchState['pipeline_config']['cluster.EXEC_NODES']), 0, batchState['pipeline_config']['cluster.CLUSTER_CREDENTIAL'], { 'cluster.MASTER_INSTANCE_TYPE': batchState['pipeline_config']['cluster.MASTER_INSTANCE_TYPE'], 'cluster.MASTER_BID_PRICE': batchState['pipeline_config']['cluster.MASTER_BID_PRICE'], 'cluster.EXEC_INSTANCE_TYPE': batchState['pipeline_config']['cluster.EXEC_INSTANCE_TYPE'], 'cluster.EXEC_BID_PRICE': batchState['pipeline_config']['cluster.EXEC_BID_PRICE'] }) yield _updateTask(batchState, lambda t: t.update(completedTasks=0, numTasks=9)) yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed startcluster').progress()) batchState['pipeline_state'] = REMOTE_LOCAL_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REMOTE_LOCAL_TRANSFER_STATE: _log(batchState, 'Pipeline is in REMOTE_LOCAL_TRANSFER') _log(batchState, 'Making sure cluster is exists in some form') cluster = yield clusters_client.loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') if cluster['state'] == 'unresponsive': _log(batchState, 'Pipeline is unresponsive, erroring and restarting') raise Exception('Cluster is not responsive') yield state.prerunQueue.addWithDeferred(_remoteLocalTransfer, batchState) yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed remote_local_transfer'). progress()) batchState['pipeline_state'] = DECRYPT_STATE state.updateBatchState() if batchState['pipeline_state'] == DECRYPT_STATE: _log(batchState, 'Pipeline is in DECRYPT') cluster = yield loadCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') tag = yield tags_client.loadTag( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest', _decryptTagName(batchState)) conf = config.configFromStream(open('/tmp/machine.conf')) yield ssh.runProcessSSH(cluster['master']['public_dns'], 'mkdir -p /mnt/lgt_decrypt', stdoutf=None, stderrf=None, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), log=True) yield rsync.rsyncTo( cluster['master']['public_dns'], batchState['pipeline_config']['params.DECRYPT_SCRIPT'], '/mnt/', options=conf('rsync.options'), user=conf('rsync.user'), log=True) for f in tag['files']: decryptCmd = ' '.join([ os.path.join( '/mnt', os.path.basename(batchState['pipeline_config'] ['params.DECRYPT_SCRIPT'])), f, '-out-dir', '/mnt/lgt_decrypt', '-remove-encrypted', '-password', batchState['pipeline_config']['params.DECRYPT_PASSWORD'] ]) yield ssh.getOutput(cluster['master']['public_dns'], decryptCmd, sshUser=conf('ssh.user'), sshFlags=conf('ssh.options'), expected=[0, 253], log=True) tag = yield tags_client.tagData( host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', action='overwrite', tagName=_decryptTagName(batchState), files=['/mnt/lgt_decrypt'], metadata={}, recursive=True, expand=False, compressDir=None) _log( batchState, 'Waiting for tagging of %s to complete - %s' % (_decryptTagName(batchState), tag['task_name'])) yield _blockOnTask( tag['task_name'], cluster=batchState['pipeline_config']['cluster.CLUSTER_NAME']) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed decrypt').progress()) batchState['pipeline_state'] = REFERENCE_TRANSFER_STATE state.updateBatchState() if batchState['pipeline_state'] == REFERENCE_TRANSFER_STATE: _log(batchState, 'Pipeline is in REFERENCE_TRANSFER state') transfers = [] tags = (batchState['pipeline_config']['input.REF_TAG1'].split(',') + batchState['pipeline_config']['input.REF_TAG2'].split(',')) for tag in tags: tag = tag.strip() output = yield _getOutput(batchState, [ 'vp-transfer-dataset', '-t', '--tag-name=' + tag, '--dst-cluster=' + batchState['pipeline_config']['cluster.CLUSTER_NAME'] ], log=True) transfers.append(output['stdout'].strip()) for task in transfers: yield _blockOnTask(task) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed reference_transfer').progress()) batchState['pipeline_state'] = RUN_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUN_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUN_PIPELINE state') batchState['pipeline_config']['input.INPUT_TAG'] = _decryptTagName( batchState) pipeline = yield pipelines_client.runPipeline( host='localhost', clusterName=batchState['pipeline_config']['cluster.CLUSTER_NAME'], userName='******', parentPipeline=batchState['pipeline_name'], bareRun=True, queue=state.innerPipelineQueue(), config=batchState['pipeline_config'], overwrite=True) batchState['pipeline_task'] = pipeline['task_name'] yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed run pipeline').progress()) batchState['pipeline_state'] = RUNNING_PIPELINE_STATE state.updateBatchState() if batchState['pipeline_state'] == RUNNING_PIPELINE_STATE: _log(batchState, 'Pipeline is in RUNNING_PIPELINE state') _monitorPipeline(batchState) yield _waitForPipeline(batchState) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed running pipeline').progress()) batchState['pipeline_state'] = HARVEST_STATE state.updateBatchState() if batchState['pipeline_state'] == HARVEST_STATE: _log(batchState, 'Pipeline is in HARVEST state') # Using prerunqueue because we want everything here serialized yield state.prerunQueue.addWithDeferred(_harvestTransfer, batchState) yield _updateTask( batchState, lambda t: t.addMessage(tasks.task.MSG_SILENT, 'Completed harvest').progress()) batchState['pipeline_state'] = SHUTDOWN_STATE state.updateBatchState() if batchState['pipeline_state'] == SHUTDOWN_STATE: _log(batchState, 'Pipeline is in SHUTDOWN state') if 'add_instances_task' in batchState: try: yield _blockOnTask(batchState['add_instances_task'], cluster=batchState['pipeline_config'] ['cluster.CLUSTER_NAME']) except Exception, err: logging.errorPrint(str(err)) log.err(err) yield clusters_client.terminateCluster( 'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'], 'guest') yield _updateTask( batchState, lambda t: t.addMessage( tasks.task.MSG_SILENT, 'Completed shutdown').progress()) batchState['pipeline_state'] = COMPLETED_STATE batchState['state'] = COMPLETED_STATE state.updateBatchState()
def _uploadTag(request): localTag = yield request.state.tagPersist.loadTag(request.body['tag_name']) srcClusters = yield www_clusters.listClusters( 'localhost', {'cluster_name': request.body['src_cluster']}, request.body['user_name']) srcCluster = srcClusters[0] dstClusters = yield www_clusters.listClusters( 'localhost', {'cluster_name': request.body['dst_cluster']}, request.body['user_name']) dstCluster = dstClusters[0] # We want the trailing '/' so everyone knows it's a directory dstTagPath = os.path.join(dstCluster['config']['dirs.upload_dir'], localTag.tagName) + '/' # Change dstTagPath to use /mnt/nostaging instead of mnt/staging as the destination if the # tag has a nostaging metadata value #if localTag.metadata['nostaging']: # dstTagPath = os.path.join(dstCluster['config']['dirs.nostaging_upload_dir'], localTag.tagName) + '/' if request.body['dst_type'] == 'local': upload_dir = '/mnt/nostaging/' if 'dirs.nostaging_upload_dir' in dstCluster['config']: upload_dir = dstCluster['config']['dirs.nostaging_upload_dir'] yield _makeDirsOnCluster(dstCluster, [upload_dir]) dstTagPath = os.path.join(upload_dir, localTag.tagName) + '/' baseDirFiles, nonBaseDirFiles = _partitionFiles( localTag.files, localTag.metadata['tag_base_dir']) if baseDirFiles: yield rsync.rsyncTo(dstCluster['master']['public_dns'], localTag.metadata['tag_base_dir'], dstTagPath, baseDirFiles, srcCluster['config']['rsync.options'], srcCluster['config']['rsync.user'], log=True) if nonBaseDirFiles: yield rsync.rsyncTo(dstCluster['master']['public_dns'], '/', dstTagPath, nonBaseDirFiles, srcCluster['config']['rsync.options'], srcCluster['config']['rsync.user'], log=True) remoteFiles = ([os.path.join(dstTagPath, f) for f in baseDirFiles] + [ os.path.join(dstTagPath, _makePathRelative(f)) for f in nonBaseDirFiles ]) metadata = localTag.metadata if metadata.get('urls', []) and not metadata.get('urls_realized', False): tag = yield _realizeUrls(request) remoteFiles.extend(tag['files']) metadata = func.updateDict(metadata, {'urls_realized': True}) # If we are dealing with an adhoc cluster here there is a chance that # we are attempting to transfer to a local VM's shared folder which does # support chown/chmod so we will skip this step if dstCluster['master']['instance_type'] is not None: yield ssh.runProcessSSH( dstCluster['master']['public_dns'], 'chown -R %s %s' % (dstCluster['config']['vappio.user'], dstTagPath), None, log.err, srcCluster['config']['ssh.user'], srcCluster['config']['ssh.options']) defer.returnValue( persist.Tag(tagName=localTag.tagName, files=remoteFiles, metadata=func.updateDict(metadata, {'tag_base_dir': dstTagPath}), phantom=localTag.phantom, taskName=None))