def _startRemotePipeline(request):
clusters = yield clusters_client.listClusters(
{'cluster_name': request.body['cluster']},
request.body['user_name'])
cluster = clusters[0]
# Forward the request on to the remote cluster, set parent_pipeline to None
ret = yield pipelines_www_client.runPipeline(
cluster['master']['public_dns'], 'local',
request.body['user_name'], None, request.body['bare_run'],
request.body.get('queue',
'pipelinewrapper.q'), request.body['config'],
request.body.get('overwrite', False))
defer.returnValue(ret)
def _(request):
if request.body['cluster'] == 'local':
defer.returnValue(request)
else:
clusters = yield clusters_client.listClusters({'cluster_name': request.body['cluster']},
request.body['user_name'])
cluster = clusters[0]
ret = yield http.performQuery(cluster['master']['public_dns'],
url,
func.updateDict(request.body, {'cluster': 'local'}),
timeout=10,
tries=3)
defer_pipe.emit(request.update(response=ret))
def _startRemotePipeline(request):
clusters = yield clusters_client.listClusters({'cluster_name': request.body['cluster']},
request.body['user_name'])
cluster = clusters[0]
# Forward the request on to the remote cluster, set parent_pipeline to None
ret = yield pipelines_www_client.runPipeline(cluster['master']['public_dns'],
'local',
request.body['user_name'],
None,
request.body['bare_run'],
request.body.get('queue', 'pipelinewrapper.q'),
request.body['config'],
request.body.get('overwrite', False))
defer.returnValue(ret)
def getKnownInstances(credentialName):
clusters = yield clusters_client.listClusters({}, 'guest')
clustersOnCred = [
cluster for cluster in clusters
if cluster['cred_name'] == credentialName
]
knownInstances = set()
for cluster in clustersOnCred:
allClusterInstances = (
[cluster['master']['instance_id']] +
[i['instance_id'] for i in cluster['exec_nodes']] +
[i['instance_id'] for i in cluster['data_nodes']])
for i in allClusterInstances:
knownInstances.add(i)
defer.returnValue(knownInstances)
def getKnownInstances(credentialName):
clusters = yield clusters_client.listClusters({}, 'guest')
clustersOnCred = [cluster
for cluster in clusters
if cluster['cred_name'] == credentialName]
knownInstances = set()
for cluster in clustersOnCred:
allClusterInstances = ([cluster['master']['instance_id']] +
[i['instance_id']
for i in cluster['exec_nodes']] +
[i['instance_id']
for i in cluster['data_nodes']])
for i in allClusterInstances:
knownInstances.add(i)
defer.returnValue(knownInstances)
def _(request):
if request.body['cluster'] == 'local':
defer.returnValue(request)
else:
clusters = yield clusters_client.listClusters(
{'cluster_name': request.body['cluster']},
request.body['user_name'])
cluster = clusters[0]
ret = yield http.performQuery(cluster['master']['public_dns'],
url,
func.updateDict(
request.body,
{'cluster': 'local'}),
timeout=10,
tries=3)
defer_pipe.emit(request.update(response=ret))
def _loopSupervisorNoThrow(state):
localClusters = yield clusters_client.listClusters(
{'cluster_name': 'local'}, None)
localCluster = localClusters[0]
loadAverages = os.getloadavg()
oneMinuteLoad = loadAverages[0]
if (oneMinuteLoad > LOAD_THRESHOLD and state.master.execSlots() > 0
and len(localCluster['exec_nodes']) > 0):
execSlots = state.master.execSlots()
execSlots -= 1
yield sge_queue.setSlotsForQueue(EXEC_QUEUE, state.hostname, execSlots)
state.master.setExecSlots(execSlots)
log.msg('LOAD: Reduced exec slots by one')
elif (oneMinuteLoad < LOAD_THRESHOLD and state.master.execSlots() == 0
and len(localCluster['exec_nodes']) == 0):
execSlots = state.master.execSlots()
execSlots += 1
yield sge_queue.setSlotsForQueue(EXEC_QUEUE, state.hostname, execSlots)
state.master.setExecSlots(execSlots)
log.msg('LOAD: Increased exec slots by one')
def _loopSupervisorNoThrow(state):
localClusters = yield clusters_client.listClusters({'cluster_name': 'local'}, None)
localCluster = localClusters[0]
loadAverages = os.getloadavg()
oneMinuteLoad = loadAverages[0]
if (oneMinuteLoad > LOAD_THRESHOLD and
state.master.execSlots() > 0 and
len(localCluster['exec_nodes']) > 0):
execSlots = state.master.execSlots()
execSlots -= 1
yield sge_queue.setSlotsForQueue(EXEC_QUEUE, state.hostname, execSlots)
state.master.setExecSlots(execSlots)
log.msg('LOAD: Reduced exec slots by one')
elif (oneMinuteLoad < LOAD_THRESHOLD and
state.master.execSlots() == 0 and
len(localCluster['exec_nodes']) == 0):
execSlots = state.master.execSlots()
execSlots += 1
yield sge_queue.setSlotsForQueue(EXEC_QUEUE, state.hostname, execSlots)
state.master.setExecSlots(execSlots)
log.msg('LOAD: Increased exec slots by one')
