def _startRemotePipeline(request): clusters = yield clusters_client.listClusters( {'cluster_name': request.body['cluster']}, request.body['user_name']) cluster = clusters[0] # Forward the request on to the remote cluster, set parent_pipeline to None ret = yield pipelines_www_client.runPipeline( cluster['master']['public_dns'], 'local', request.body['user_name'], None, request.body['bare_run'], request.body.get('queue', 'pipelinewrapper.q'), request.body['config'], request.body.get('overwrite', False)) defer.returnValue(ret)
def _(request): if request.body['cluster'] == 'local': defer.returnValue(request) else: clusters = yield clusters_client.listClusters({'cluster_name': request.body['cluster']}, request.body['user_name']) cluster = clusters[0] ret = yield http.performQuery(cluster['master']['public_dns'], url, func.updateDict(request.body, {'cluster': 'local'}), timeout=10, tries=3) defer_pipe.emit(request.update(response=ret))
def _startRemotePipeline(request): clusters = yield clusters_client.listClusters({'cluster_name': request.body['cluster']}, request.body['user_name']) cluster = clusters[0] # Forward the request on to the remote cluster, set parent_pipeline to None ret = yield pipelines_www_client.runPipeline(cluster['master']['public_dns'], 'local', request.body['user_name'], None, request.body['bare_run'], request.body.get('queue', 'pipelinewrapper.q'), request.body['config'], request.body.get('overwrite', False)) defer.returnValue(ret)
def getKnownInstances(credentialName): clusters = yield clusters_client.listClusters({}, 'guest') clustersOnCred = [ cluster for cluster in clusters if cluster['cred_name'] == credentialName ] knownInstances = set() for cluster in clustersOnCred: allClusterInstances = ( [cluster['master']['instance_id']] + [i['instance_id'] for i in cluster['exec_nodes']] + [i['instance_id'] for i in cluster['data_nodes']]) for i in allClusterInstances: knownInstances.add(i) defer.returnValue(knownInstances)
def getKnownInstances(credentialName): clusters = yield clusters_client.listClusters({}, 'guest') clustersOnCred = [cluster for cluster in clusters if cluster['cred_name'] == credentialName] knownInstances = set() for cluster in clustersOnCred: allClusterInstances = ([cluster['master']['instance_id']] + [i['instance_id'] for i in cluster['exec_nodes']] + [i['instance_id'] for i in cluster['data_nodes']]) for i in allClusterInstances: knownInstances.add(i) defer.returnValue(knownInstances)
def _(request): if request.body['cluster'] == 'local': defer.returnValue(request) else: clusters = yield clusters_client.listClusters( {'cluster_name': request.body['cluster']}, request.body['user_name']) cluster = clusters[0] ret = yield http.performQuery(cluster['master']['public_dns'], url, func.updateDict( request.body, {'cluster': 'local'}), timeout=10, tries=3) defer_pipe.emit(request.update(response=ret))
def _loopSupervisorNoThrow(state): localClusters = yield clusters_client.listClusters( {'cluster_name': 'local'}, None) localCluster = localClusters[0] loadAverages = os.getloadavg() oneMinuteLoad = loadAverages[0] if (oneMinuteLoad > LOAD_THRESHOLD and state.master.execSlots() > 0 and len(localCluster['exec_nodes']) > 0): execSlots = state.master.execSlots() execSlots -= 1 yield sge_queue.setSlotsForQueue(EXEC_QUEUE, state.hostname, execSlots) state.master.setExecSlots(execSlots) log.msg('LOAD: Reduced exec slots by one') elif (oneMinuteLoad < LOAD_THRESHOLD and state.master.execSlots() == 0 and len(localCluster['exec_nodes']) == 0): execSlots = state.master.execSlots() execSlots += 1 yield sge_queue.setSlotsForQueue(EXEC_QUEUE, state.hostname, execSlots) state.master.setExecSlots(execSlots) log.msg('LOAD: Increased exec slots by one')
def _loopSupervisorNoThrow(state): localClusters = yield clusters_client.listClusters({'cluster_name': 'local'}, None) localCluster = localClusters[0] loadAverages = os.getloadavg() oneMinuteLoad = loadAverages[0] if (oneMinuteLoad > LOAD_THRESHOLD and state.master.execSlots() > 0 and len(localCluster['exec_nodes']) > 0): execSlots = state.master.execSlots() execSlots -= 1 yield sge_queue.setSlotsForQueue(EXEC_QUEUE, state.hostname, execSlots) state.master.setExecSlots(execSlots) log.msg('LOAD: Reduced exec slots by one') elif (oneMinuteLoad < LOAD_THRESHOLD and state.master.execSlots() == 0 and len(localCluster['exec_nodes']) == 0): execSlots = state.master.execSlots() execSlots += 1 yield sge_queue.setSlotsForQueue(EXEC_QUEUE, state.hostname, execSlots) state.master.setExecSlots(execSlots) log.msg('LOAD: Increased exec slots by one')