def _handleTerminateInstances(request): yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.setState(tasks_tx.task.TASK_RUNNING)) persistManager = request.state.persistManager cluster = yield persistManager.loadCluster(request.body['cluster_name'], request.body['user_name']) credClient = cred_client.CredentialClient(cluster.credName, request.mq, request.state.conf) if request.body['cluster_name'] != 'local': try: remoteTaskName = yield clusters_client_www.terminateInstances( cluster.master['public_dns'], 'local', request.body['user_name'], request.body['by_attribute'], request.body['attribute_values']) localTask = yield tasks_tx.loadTask(request.body['task_name']) yield tasks_tx.blockOnTaskAndForward('localhost', request.body['cluster_name'], remoteTaskName, localTask) except: yield terminateInstancesByAttribute( persistManager, credClient, request.body['cluster_name'], request.body['user_name'], request.body['by_attribute'], request.body['attribute_values']) else: cl = yield terminateInstancesByAttribute( persistManager, credClient, 'local', None, request.body['by_attribute'], request.body['attribute_values']) yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.progress()) defer.returnValue(request)
def terminateRemoteCluster(request): persistManager = request.state.persistManager cluster = yield persistManager.loadCluster(request.body["cluster_name"], request.body["user_name"]) authToken = auth_token.generateToken(cluster.config("cluster.cluster_public_key")) credClient = cred_client.CredentialClient(cluster.credName, request.mq, request.state.conf) try: if cluster.master: wwwTerminateCluster = clusters_client_www.terminateCluster remoteTaskName = yield wwwTerminateCluster(cluster.master["public_dns"], "local", None, authToken) localTask = yield tasks_tx.loadTask(request.body["task_name"]) yield tasks_tx.blockOnTaskAndForward("localhost", request.body["cluster_name"], remoteTaskName, localTask) except errors.RemoteError, err: # If the error is not an auth token one then kill it # otherwise it means we think we own a cluster that # we don't # # In this case another part of the system is in charge # of forgetting about the clusters we shouldn't know if err.name != "igs.utils.auth_token.AuthTokenError": log.err(err) yield terminateCluster(credClient, persistManager, request.body["cluster_name"], request.body["user_name"]) else: raise
def terminateRemoteCluster(request): persistManager = request.state.persistManager cluster = yield persistManager.loadCluster(request.body['cluster_name'], request.body['user_name']) authToken = auth_token.generateToken( cluster.config('cluster.cluster_public_key')) credClient = cred_client.CredentialClient(cluster.credName, request.mq, request.state.conf) try: if cluster.master: wwwTerminateCluster = clusters_client_www.terminateCluster remoteTaskName = yield wwwTerminateCluster( cluster.master['public_dns'], 'local', None, authToken) localTask = yield tasks_tx.loadTask(request.body['task_name']) yield tasks_tx.blockOnTaskAndForward('localhost', request.body['cluster_name'], remoteTaskName, localTask) except errors.RemoteError, err: # If the error is not an auth token one then kill it # otherwise it means we think we own a cluster that # we don't # # In this case another part of the system is in charge # of forgetting about the clusters we shouldn't know if err.name != 'igs.utils.auth_token.AuthTokenError': log.err(err) yield terminateCluster(credClient, persistManager, request.body['cluster_name'], request.body['user_name']) else: raise
def _tagToDictAndCache(self, aspect, tag): if tag.taskName: try: t = yield tasks_tx.loadTask(tag.taskName) state = t.state except: state = None else: state = None d = {} d.update({"files": tag.files, "metadata": tag.metadata}) d.update( { "tag_name": tag.tagName, "file_count": len(tag.files), "pipelines": [], "task_name": tag.taskName, "state": state, "phantom": config.configToDict(tag.phantom) if tag.phantom else None, } ) yield self.cache.save(d) self.changed(aspect, d)
def pipelineToDict(self, pipeline): protocolConf = protocol_format.load(self.machineConf, pipeline.config("pipeline.PIPELINE_TEMPLATE")) inputTagsList = [ pipeline.config(k).split(",") for k, v in protocolConf if v.get("type").split()[0] in ["dataset", "blastdb_dataset", "paired_dataset", "singleton_dataset"] and pipeline.config(k) ] inputTags = [] for i in inputTagsList: inputTags.extend(i) possibleOutputTags = set( [ pipeline.pipelineName + "_" + t.strip() for t in pipeline.config("output.TAGS_TO_DOWNLOAD", default="").split(",") ] ) query = [{"tag_name": t} for t in possibleOutputTags] tags = yield www_tags.loadTagsBy("localhost", "local", pipeline.userName, {"$or": query}, False) tags = set([t["tag_name"] for t in tags]) outputTags = list(tags & possibleOutputTags) pipelineTask = yield tasks_tx.loadTask(pipeline.taskName) pipelineWrapper = pipeline_misc.determineWrapper( self.machineConf, pipeline.config("pipeline.PIPELINE_TEMPLATE") ) pipelineDict = { "pipeline_id": pipeline.pipelineId, "pipeline_name": pipeline.pipelineName, "user_name": pipeline.userName, "wrapper": pipeline.protocol == pipelineWrapper, "protocol": pipeline.config("pipeline.PIPELINE_TEMPLATE"), "checksum": pipeline.checksum, "task_name": pipeline.taskName, "queue": pipeline.queue, "children": pipeline.children, "state": pipelineTask.state, "num_steps": pipelineTask.numTasks, "num_complete": pipelineTask.completedTasks, "input_tags": inputTags, "output_tags": outputTags, "pipeline_desc": pipeline.config("pipeline.PIPELINE_DESC", default=""), "config": config.configToDict(pipeline.config, lazy=True), } defer.returnValue(pipelineDict)
def pipelineToDict(self, pipeline): protocolConf = protocol_format.load(self.machineConf, pipeline.config('pipeline.PIPELINE_TEMPLATE')) inputTagsList = [pipeline.config(k).split(',') for k, v in protocolConf if v.get('type').split()[0] in ['dataset', 'blastdb_dataset', 'paired_dataset', 'singleton_dataset'] and pipeline.config(k)] inputTags = [] for i in inputTagsList: inputTags.extend(i) possibleOutputTags = set([pipeline.pipelineName + '_' + t.strip() for t in pipeline.config('output.TAGS_TO_DOWNLOAD', default='').split(',')]) query = [{'tag_name': t} for t in possibleOutputTags] tags = yield www_tags.loadTagsBy('localhost', 'local', pipeline.userName, {'$or': query}, False) tags = set([t['tag_name'] for t in tags]) outputTags = list(tags & possibleOutputTags) pipelineTask = yield tasks_tx.loadTask(pipeline.taskName) pipelineWrapper = pipeline_misc.determineWrapper(self.machineConf, pipeline.config('pipeline.PIPELINE_TEMPLATE')) pipelineDict = {'pipeline_id': pipeline.pipelineId, 'pipeline_name': pipeline.pipelineName, 'user_name': pipeline.userName, 'wrapper': pipeline.protocol == pipelineWrapper, 'protocol': pipeline.config('pipeline.PIPELINE_TEMPLATE'), 'checksum': pipeline.checksum, 'task_name': pipeline.taskName, 'queue': pipeline.queue, 'children': pipeline.children, 'state': pipelineTask.state, 'num_steps': pipelineTask.numTasks, 'num_complete': pipelineTask.completedTasks, 'input_tags': inputTags, 'output_tags': outputTags, 'pipeline_desc': pipeline.config('pipeline.PIPELINE_DESC', default=''), 'config': config.configToDict(pipeline.config, lazy=True), } defer.returnValue(pipelineDict)
def _handleTerminateInstances(request): yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.setState(tasks_tx.task.TASK_RUNNING)) persistManager = request.state.persistManager cluster = yield persistManager.loadCluster(request.body['cluster_name'], request.body['user_name']) credClient = cred_client.CredentialClient(cluster.credName, request.mq, request.state.conf) if request.body['cluster_name'] != 'local': try: remoteTaskName = yield clusters_client_www.terminateInstances( cluster.master['public_dns'], 'local', request.body['user_name'], request.body['by_attribute'], request.body['attribute_values']) localTask = yield tasks_tx.loadTask(request.body['task_name']) yield tasks_tx.blockOnTaskAndForward('localhost', request.body['cluster_name'], remoteTaskName, localTask) except: yield terminateInstancesByAttribute(persistManager, credClient, request.body['cluster_name'], request.body['user_name'], request.body['by_attribute'], request.body['attribute_values']) else: cl = yield terminateInstancesByAttribute(persistManager, credClient, 'local', None, request.body['by_attribute'], request.body['attribute_values']) yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.progress()) defer.returnValue(request)
def _tagToDictAndCache(self, aspect, tag): if tag.taskName: try: t = yield tasks_tx.loadTask(tag.taskName) state = t.state except: state = None else: state = None d = {} d.update({'files': tag.files, 'metadata': tag.metadata}) d.update({'tag_name': tag.tagName, 'file_count': len(tag.files), 'pipelines': [], 'task_name': tag.taskName, 'state': state, 'phantom': config.configToDict(tag.phantom) if tag.phantom else None}) yield self.cache.save(d) self.changed(aspect, d)
def _realizeUrls(request): localTag = yield request.state.tagPersist.loadTag(request.body['tag_name']) # If we have urls we create a fake phantom tag fakePhantom = {'cluster.ALL.command': 'reliableDownloader.py -m 300 -t 20 -b ${base_dir} ' + ' '.join(localTag.metadata['urls'])} taskName = yield www_tags.realizePhantom('localhost', request.body['dst_cluster'], request.body['user_name'], localTag.tagName, fakePhantom, func.updateDict(localTag.metadata, {'urls_realized': True})) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost', request.body['dst_cluster'], taskName, localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.setState(tasks_tx.task.TASK_FAILED)) raise RealizePhantomError(request.body['tag_name']) if request.body['dst_cluster'] == 'local': yield tag_mq_data.tagData(request.state, request.body['tag_name'], request.body['task_name'], files=localTag.files, action=tag_mq_data.ACTION_APPEND, metadata={}, recursive=False, expand=False, compressDir=None) else: localTask = yield www_tags.tagData('localhost', request.body['dst_cluster'], request.body['user_name'], action=tag_mq_data.ACTION_APPEND, tagName=localTag.tagName, files=localTag.files, metadata={}, recursive=False, expand=False, compressDir=None) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost', request.body['dst_cluster'], taskName, localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.setState(tasks_tx.task.TASK_FAILED)) raise RealizePhantomError(request.body['tag_name']) # Load the tag up and return it so we can have the files it created tag = yield www_tags.loadTag('localhost', request.body['dst_cluster'], request.body['user_name'], request.body['tag_name']) defer.returnValue(tag)
def _handleTransferTag(request): yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=2)) srcTag = yield www_tags.loadTag('localhost', request.body['src_cluster'], request.body['user_name'], request.body['tag_name']) if not srcTag['phantom'] and (request.body['src_cluster'] != 'local' or request.body['dst_cluster'] != 'local'): if request.body['src_cluster'] == 'local': tag = yield _uploadTag(request) elif request.body['dst_cluster'] == 'local': tag = yield _downloadTag(request) else: raise NoLocalClusterError('Source cluster or destination cluster must be local') yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.progress()) if request.body.get('compress', False) or request.body.get('compress_dir', False): defaultDir = '/mnt/output' if request.body['dst_cluster'] == 'local' else tag.metadata['tag_base_dir'] compressDir = request.body.get('compress_dir') if request.body.get('compress_dir', False) else defaultDir else: compressDir = None if request.body['dst_cluster'] == 'local': yield tag_mq_data.tagData(request.state, request.body['tag_name'], request.body['task_name'], files=tag.files, action=tag_mq_data.ACTION_OVERWRITE, metadata=tag.metadata, recursive=False, expand=False, compressDir=compressDir) else: newTag = yield www_tags.tagData('localhost', request.body['dst_cluster'], request.body['user_name'], action=tag_mq_data.ACTION_OVERWRITE, tagName=tag.tagName, files=tag.files, metadata=tag.metadata, recursive=False, expand=False, compressDir=compressDir) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost', request.body['dst_cluster'], newTag['task_name'], localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.setState(tasks_tx.task.TASK_FAILED)) raise TransferTagError(request.body['tag_name']) yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.progress()) elif not srcTag['phantom'] and srcTag['metadata'].get('urls', []) and not srcTag['metadata'].get('urls_realized', False): # It's a local to local but we have urls and haven't realized them yield _realizeUrls(request) yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.progress(2)) elif srcTag['phantom']: # Upload the depends file srcClusters = yield www_clusters.listClusters('localhost', {'cluster_name': request.body['src_cluster']}, request.body['user_name']) srcCluster = srcClusters[0] dstClusters = yield www_clusters.listClusters('localhost', {'cluster_name': request.body['dst_cluster']}, request.body['user_name']) dstCluster = dstClusters[0] dependsOn = srcTag['phantom'].get('depends_on', '').split() yield rsync.rsyncTo(dstCluster['master']['public_dns'], '/', '/', dependsOn, srcCluster['config']['rsync.options'], srcCluster['config']['rsync.user'], log=True) taskName = yield www_tags.realizePhantom('localhost', request.body['dst_cluster'], request.body['user_name'], srcTag['tag_name'], srcTag['phantom'], srcTag['metadata']) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward('localhost', request.body['dst_cluster'], taskName, localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.setState(tasks_tx.task.TASK_FAILED)) raise RealizePhantomError(request.body['tag_name']) yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.update(numTasks=1).progress()) else: yield tag_mq_data.tagData(request.state, request.body['tag_name'], request.body['task_name'], files=[], action=tag_mq_data.ACTION_APPEND, metadata={}, recursive=False, expand=False, compressDir='/mnt/output' if request.body.get('compress', False) else None) yield tasks_tx.updateTask(request.body['task_name'], lambda t : t.progress(2)) defer.returnValue(request)
log.err('STARTCLUSETER: Failed') log.err(err) cluster = yield request.state.persistManager.loadCluster( request.body['cluster_name'], request.body['user_name']) cluster = cluster.setState(cluster.FAILED) yield defer_utils.sleep(120)() yield request.state.persistManager.removeCluster( request.body['cluster_name'], request.body['user_name']) raise err if request.body['num_exec'] > 0 or request.body['num_data'] > 0: addInstancesTaskName = yield clusters_client_www.addInstances( 'localhost', request.body['cluster_name'], request.body['user_name'], request.body['num_exec'], request.body['num_data']) localTask = yield tasks_tx.loadTask(request.body['task_name']) yield tasks_tx.blockOnTaskForward('localhost', request.body['cluster_name'], addInstancesTaskName, localTask) defer.returnValue(request) @defer.inlineCallbacks def handleStartCluster(request): ret = yield request.state.clusterLocks.run( (request.body['cluster_name'], request.body['user_name']), _handleStartCluster, request) defer.returnValue(ret)
cluster = yield request.state.persistManager.loadCluster(request.body['cluster_name'], request.body['user_name']) cluster = cluster.setState(cluster.FAILED) yield defer_utils.sleep(120)() yield request.state.persistManager.removeCluster(request.body['cluster_name'], request.body['user_name']) raise err if request.body['num_exec'] > 0 or request.body['num_data'] > 0: addInstancesTaskName = yield clusters_client_www.addInstances('localhost', request.body['cluster_name'], request.body['user_name'], request.body['num_exec'], request.body['num_data']) localTask = yield tasks_tx.loadTask(request.body['task_name']) yield tasks_tx.blockOnTaskForward('localhost', request.body['cluster_name'], addInstancesTaskName, localTask) defer.returnValue(request) @defer.inlineCallbacks def handleStartCluster(request): ret = yield request.state.clusterLocks.run((request.body['cluster_name'], request.body['user_name']), _handleStartCluster, request) defer.returnValue(ret)
def _realizeUrls(request): localTag = yield request.state.tagPersist.loadTag(request.body['tag_name']) # If we have urls we create a fake phantom tag fakePhantom = { 'cluster.ALL.command': 'reliableDownloader.py -m 300 -t 20 -b ${base_dir} ' + ' '.join(localTag.metadata['urls']) } taskName = yield www_tags.realizePhantom( 'localhost', request.body['dst_cluster'], request.body['user_name'], localTag.tagName, fakePhantom, func.updateDict(localTag.metadata, {'urls_realized': True})) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward( 'localhost', request.body['dst_cluster'], taskName, localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask( request.body['task_name'], lambda t: t.setState(tasks_tx.task.TASK_FAILED)) raise RealizePhantomError(request.body['tag_name']) if request.body['dst_cluster'] == 'local': yield tag_mq_data.tagData(request.state, request.body['tag_name'], request.body['task_name'], files=localTag.files, action=tag_mq_data.ACTION_APPEND, metadata={}, recursive=False, expand=False, compressDir=None) else: localTask = yield www_tags.tagData('localhost', request.body['dst_cluster'], request.body['user_name'], action=tag_mq_data.ACTION_APPEND, tagName=localTag.tagName, files=localTag.files, metadata={}, recursive=False, expand=False, compressDir=None) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward( 'localhost', request.body['dst_cluster'], taskName, localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask( request.body['task_name'], lambda t: t.setState(tasks_tx.task.TASK_FAILED)) raise RealizePhantomError(request.body['tag_name']) # Load the tag up and return it so we can have the files it created tag = yield www_tags.loadTag('localhost', request.body['dst_cluster'], request.body['user_name'], request.body['tag_name']) defer.returnValue(tag)
def _handleTransferTag(request): yield tasks_tx.updateTask( request.body['task_name'], lambda t: t.setState(tasks_tx.task.TASK_RUNNING).update(numTasks=2)) srcTag = yield www_tags.loadTag('localhost', request.body['src_cluster'], request.body['user_name'], request.body['tag_name']) if not srcTag['phantom'] and (request.body['src_cluster'] != 'local' or request.body['dst_cluster'] != 'local'): if request.body['src_cluster'] == 'local': tag = yield _uploadTag(request) elif request.body['dst_cluster'] == 'local': tag = yield _downloadTag(request) else: raise NoLocalClusterError( 'Source cluster or destination cluster must be local') yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.progress()) if request.body.get('compress', False) or request.body.get( 'compress_dir', False): defaultDir = '/mnt/output' if request.body[ 'dst_cluster'] == 'local' else tag.metadata['tag_base_dir'] compressDir = request.body.get('compress_dir') if request.body.get( 'compress_dir', False) else defaultDir else: compressDir = None if request.body['dst_cluster'] == 'local': yield tag_mq_data.tagData(request.state, request.body['tag_name'], request.body['task_name'], files=tag.files, action=tag_mq_data.ACTION_OVERWRITE, metadata=tag.metadata, recursive=False, expand=False, compressDir=compressDir) else: newTag = yield www_tags.tagData( 'localhost', request.body['dst_cluster'], request.body['user_name'], action=tag_mq_data.ACTION_OVERWRITE, tagName=tag.tagName, files=tag.files, metadata=tag.metadata, recursive=False, expand=False, compressDir=compressDir) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward( 'localhost', request.body['dst_cluster'], newTag['task_name'], localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask( request.body['task_name'], lambda t: t.setState(tasks_tx.task.TASK_FAILED)) raise TransferTagError(request.body['tag_name']) yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.progress()) elif not srcTag['phantom'] and srcTag['metadata'].get( 'urls', []) and not srcTag['metadata'].get('urls_realized', False): # It's a local to local but we have urls and haven't realized them yield _realizeUrls(request) yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.progress(2)) elif srcTag['phantom']: # Upload the depends file srcClusters = yield www_clusters.listClusters( 'localhost', {'cluster_name': request.body['src_cluster']}, request.body['user_name']) srcCluster = srcClusters[0] dstClusters = yield www_clusters.listClusters( 'localhost', {'cluster_name': request.body['dst_cluster']}, request.body['user_name']) dstCluster = dstClusters[0] dependsOn = srcTag['phantom'].get('depends_on', '').split() yield rsync.rsyncTo(dstCluster['master']['public_dns'], '/', '/', dependsOn, srcCluster['config']['rsync.options'], srcCluster['config']['rsync.user'], log=True) taskName = yield www_tags.realizePhantom('localhost', request.body['dst_cluster'], request.body['user_name'], srcTag['tag_name'], srcTag['phantom'], srcTag['metadata']) localTask = yield tasks_tx.loadTask(request.body['task_name']) endState, tsk = yield tasks_tx.blockOnTaskAndForward( 'localhost', request.body['dst_cluster'], taskName, localTask) if endState == tasks_tx.task.TASK_FAILED: yield tasks_tx.updateTask( request.body['task_name'], lambda t: t.setState(tasks_tx.task.TASK_FAILED)) raise RealizePhantomError(request.body['tag_name']) yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.update(numTasks=1).progress()) else: yield tag_mq_data.tagData( request.state, request.body['tag_name'], request.body['task_name'], files=[], action=tag_mq_data.ACTION_APPEND, metadata={}, recursive=False, expand=False, compressDir='/mnt/output' if request.body.get('compress', False) else None) yield tasks_tx.updateTask(request.body['task_name'], lambda t: t.progress(2)) defer.returnValue(request)