def _createPipeline(request): taskName = yield tasks_tx.createTaskAndSave('runPipelines', 0) # The name of a pipeline is being stored as a checksum. Pipeline names # are arbitrary and the user will likely never know or care what it is. # The pipeline name still exists though because other tools will likely # find it useful to refer to a pipeline by a particular name, but if # we decide to change the pipeline name to something more meaningful they # won't have to chagne their code to use pipelineName instead of checksum protocol = _determineProtocol(request) if not request.body['bare_run']: request.body['config'][ 'pipeline.PIPELINE_WRAPPER_NAME'] = request.body['config'][ 'pipeline.PIPELINE_NAME'] defer.returnValue( persist.Pipeline(pipelineId=None, pipelineName=checksum, userName=request.body['user_name'], protocol=protocol, checksum=checksum, taskName=taskName, queue=request.body.get('queue', 'pipelinewrapper.q'), children=[], config=request.body['config']))
def _(request): def _forward(taskName): request.body['task_name'] = taskName request.mq.send(dstQueue, json.dumps(request.body)) return defer_pipe.ret(request.update(response=taskName)) d = tasks.createTaskAndSave(tType, numTasks) d.addCallback(_forward) return d
def handleWWWPipelineCreate(request): """ Sets the config section of a pipeline exactly to what is given Input: { cluster: string user_name: string pipeline_name : string protocol : string queue : string config: { key/value } ?pipeline_id : string ?task_name : string ?pipeline_parent : pipeline_name } Output: Pipeline """ if request.body.get('parent_pipeline'): parentPipelines = yield request.state.pipelinePersist.loadAllPipelinesBy({'pipeline_name': request.body['parent_pipeline']}, request.body['user_name']) if len(parentPipelines) == 1: parentPipeline = parentPipelines[0] else: raise Exception('More than one possible parent pipeline choice, not sure what to do here') else: parentPipeline = None if 'task_name' in request.body: taskName = request.body['task_name'] else: taskName = yield tasks.createTaskAndSave('runPipelines', 0) pipeline = persist.Pipeline(pipelineId=request.body.get('pipeline_id', None), pipelineName=request.body['pipeline_name'], userName=request.body['user_name'], protocol=request.body['protocol'], checksum=pipeline_misc.checksumInput(request.body['config']), taskName=taskName, queue=request.body['queue'], children=[], config=request.body['config']) yield request.state.pipelinePersist.savePipeline(pipeline) pipelineDict = yield request.state.pipelinesCache.pipelineToDict(pipeline) if parentPipeline: childPipeline = [('local', request.body['pipeline_name'])] parentPipeline = parentPipeline.update(children=list(set([tuple(e) for e in parentPipeline.children + childPipeline]))) yield request.state.pipelinePersist.savePipeline(parentPipeline) defer.returnValue(request.update(response=pipelineDict))
def handleWWWListAddCredentials(request): if 'credential_name' in request.body and core.keysInDict( ['credential_name', 'description', 'ctype', 'metadata'], request.body): # Users can provide a file name or the actual contents of the # certificate. if 'cert_file' in request.body: cert = open(request.body['cert_file']).read() else: cert = request.body['cert'] if 'pkey_file' in request.body: pkey = open(request.body['pkey_file']).read() else: pkey = request.body['pkey'] conf = config.configFromMap(request.body.get('conf', {}), base=config.configFromEnv()) cred = persist.createCredential(name=request.body['credential_name'], desc=request.body['description'], ctype=request.body['ctype'], cert=cert, pkey=pkey, active=True, metadata=request.body['metadata'], conf=conf) taskName = yield tasks_tx.createTaskAndSave('addCredential', 1) instantiateAndSaveCredential(taskName, cred, request.state.credentialPersist) queue.returnQueueSuccess(request.mq, request.body['return_queue'], taskName) defer.returnValue(request) elif 'credential_name' not in request.body: credentials = request.state.credentialsCache.getAllCredentials() credentialsDicts = [{ 'name': name, 'description': c['cred_instance'].credential.desc, 'num_instances': len(c['instances']), 'ctype': c['cred_instance'].credential.getCType() } for name, c in credentials.iteritems() if ('credential_names' in request.body and name in request.body['credential_names']) or 'credential_names' not in request.body] queue.returnQueueSuccess(request.mq, request.body['return_queue'], credentialsDicts) defer.returnValue(request) else: queue.returnQueueError(request.mq, request.body['return_queue'], 'Unknown credential query') raise UnknownRequestError(str(request.body))
def _createPipeline(request): taskName = yield tasks_tx.createTaskAndSave('runPipelines', 0) # The name of a pipeline is being stored as a checksum. Pipeline names # are arbitrary and the user will likely never know or care what it is. # The pipeline name still exists though because other tools will likely # find it useful to refer to a pipeline by a particular name, but if # we decide to change the pipeline name to something more meaningful they # won't have to chagne their code to use pipelineName instead of checksum protocol = _determineProtocol(request) if not request.body['bare_run']: request.body['config']['pipeline.PIPELINE_WRAPPER_NAME'] = request.body['config']['pipeline.PIPELINE_NAME'] defer.returnValue(persist.Pipeline(pipelineId=None, pipelineName=checksum, userName=request.body['user_name'], protocol=protocol, checksum=checksum, taskName=taskName, queue=request.body.get('queue', 'pipelinewrapper.q'), children=[], config=request.body['config']))
def handleWWWListAddCredentials(request): if 'credential_name' in request.body and core.keysInDict(['credential_name', 'description', 'ctype', 'metadata'], request.body): # Users can provide a file name or the actual contents of the # certificate. if 'cert_file' in request.body: cert = open(request.body['cert_file']).read() else: cert = request.body['cert'] if 'pkey_file' in request.body: pkey = open(request.body['pkey_file']).read() else: pkey = request.body['pkey'] conf = config.configFromMap(request.body.get('conf', {}), base=config.configFromEnv()) cred = persist.createCredential(name=request.body['credential_name'], desc=request.body['description'], ctype=request.body['ctype'], cert=cert, pkey=pkey, active=True, metadata=request.body['metadata'], conf=conf) taskName = yield tasks_tx.createTaskAndSave('addCredential', 1) instantiateAndSaveCredential(taskName, cred, request.state.credentialPersist) queue.returnQueueSuccess(request.mq, request.body['return_queue'], taskName) defer.returnValue(request) elif 'credential_name' not in request.body: credentials = request.state.credentialsCache.getAllCredentials() credentialsDicts = [{'name': name, 'description': c['cred_instance'].credential.desc, 'num_instances': len(c['instances']), 'ctype': c['cred_instance'].credential.getCType()} for name, c in credentials.iteritems() if ('credential_names' in request.body and name in request.body['credential_names']) or 'credential_names' not in request.body] queue.returnQueueSuccess(request.mq, request.body['return_queue'], credentialsDicts) defer.returnValue(request) else: queue.returnQueueError(request.mq, request.body['return_queue'], 'Unknown credential query') raise UnknownRequestError(str(request.body))
def loadLocalCluster(mq, state): """ If local cluster is not present, load it """ def _credential(): if os.path.exists('/tmp/cred-info'): cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split('\t') return {'name': 'local', 'desc': 'Local credential', 'ctype': ctype, 'cert': open(cert).read(), 'pkey': open(pkey).read(), 'metadata': metadata and dict([v.split('=', 1) for v in metadata.split(',')]) or {}, 'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True)} else: return {'name': 'local', 'desc': 'Local credential', 'ctype': 'local', 'cert': None, 'pkey': None, 'metadata': {}, 'conf': config.configFromMap({})} try: cluster = yield state.persistManager.loadCluster('local', None) baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap({'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'}, base=baseConf) if (cluster.credName == 'local' and conf('MASTER_IP') not in [cluster.master['public_dns'], cluster.master['private_dns']]): master = dict(instance_id='local', ami_id=None, public_dns=conf('MASTER_IP'), private_dns=conf('MASTER_IP'), state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master).update(config=conf) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster) except persist.ClusterNotFoundError: credential = _credential() credTaskName = yield cred_client.saveCredential(credential['name'], credential['desc'], credential['ctype'], credential['cert'], credential['pkey'], credential['metadata'], credential['conf']) ## Wait for credential to be added. ## TODO: Should handle failure here yield tasks_tx.blockOnTask('localhost', 'local', credTaskName) credClient = cred_client.CredentialClient('local', mq, state.conf) ## If it isn't a local ctype then we need to wait for ## the credential to come alive if credential['ctype'] != 'local': instances = yield credClient.listInstances() else: instances = [] baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap({'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub'}, base=baseConf) cluster = persist.Cluster('local', None, 'local', conf) startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1) yield tasks_tx.updateTask(startTaskName, lambda t : t.setState(tasks_tx.task.TASK_COMPLETED).progress()) cluster = cluster.update(startTask=startTaskName) masterIp = cluster.config('MASTER_IP') masterIdx = func.find(lambda i : masterIp in [i['public_dns'], i['private_dns']], instances) if masterIdx is not None: master = instances[masterIdx] else: master = dict(instance_id='local', ami_id=None, public_dns=masterIp, private_dns=masterIp, state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master) cluster = cluster.setState(cluster.RUNNING) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster)
def loadLocalCluster(mq, state): """ If local cluster is not present, load it """ def _credential(): if os.path.exists('/tmp/cred-info'): cert, pkey, ctype, metadata = open('/tmp/cred-info').read().split( '\t') return { 'name': 'local', 'desc': 'Local credential', 'ctype': ctype, 'cert': open(cert).read(), 'pkey': open(pkey).read(), 'metadata': metadata and dict([v.split('=', 1) for v in metadata.split(',')]) or {}, 'conf': config.configFromStream(open('/tmp/machine.conf'), lazy=True) } else: return { 'name': 'local', 'desc': 'Local credential', 'ctype': 'local', 'cert': None, 'pkey': None, 'metadata': {}, 'conf': config.configFromMap({}) } try: cluster = yield state.persistManager.loadCluster('local', None) baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap( { 'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub' }, base=baseConf) if (cluster.credName == 'local' and conf('MASTER_IP') not in [ cluster.master['public_dns'], cluster.master['private_dns'] ]): master = dict(instance_id='local', ami_id=None, public_dns=conf('MASTER_IP'), private_dns=conf('MASTER_IP'), state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master).update(config=conf) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster) except persist.ClusterNotFoundError: credential = _credential() credTaskName = yield cred_client.saveCredential( credential['name'], credential['desc'], credential['ctype'], credential['cert'], credential['pkey'], credential['metadata'], credential['conf']) ## Wait for credential to be added. ## TODO: Should handle failure here yield tasks_tx.blockOnTask('localhost', 'local', credTaskName) credClient = cred_client.CredentialClient('local', mq, state.conf) ## If it isn't a local ctype then we need to wait for ## the credential to come alive if credential['ctype'] != 'local': instances = yield credClient.listInstances() else: instances = [] baseConf = config.configFromStream(open('/tmp/machine.conf'), base=config.configFromEnv()) conf = config.configFromMap( { 'config_loaded': True, 'cluster.cluster_public_key': '/mnt/keys/devel1.pem.pub' }, base=baseConf) cluster = persist.Cluster('local', None, 'local', conf) startTaskName = yield tasks_tx.createTaskAndSave('startCluster', 1) yield tasks_tx.updateTask( startTaskName, lambda t: t.setState(tasks_tx.task.TASK_COMPLETED).progress()) cluster = cluster.update(startTask=startTaskName) masterIp = cluster.config('MASTER_IP') masterIdx = func.find( lambda i: masterIp in [i['public_dns'], i['private_dns']], instances) if masterIdx is not None: master = instances[masterIdx] else: master = dict(instance_id='local', ami_id=None, public_dns=masterIp, private_dns=masterIp, state='running', key=None, index=None, instance_type=None, launch=None, availability_zone=None, monitor=None, spot_request_id=None, bid_price=None) cluster = cluster.setMaster(master) cluster = cluster.setState(cluster.RUNNING) yield state.persistManager.saveCluster(cluster) defer.returnValue(cluster)