def _sumChildrenPipelines(self, pl):
numSteps = 0
completedSteps = 0
messages = []
clusters = yield clusters_client.listClusters('localhost', {},
pl.userName)
clusterNames = set([c['cluster_name'] for c in clusters])
for cl, remotePipelineName in pl.children:
try:
if cl in clusterNames:
remotePipelines = yield pipelines_www.pipelineList(
'localhost', cl, pl.userName, remotePipelineName, True)
remotePipeline = remotePipelines[0]
remoteTask = yield tasks_www.loadTask(
'localhost', cl, pl.userName,
remotePipeline['task_name'])
numSteps += remoteTask['numTasks']
completedSteps += remoteTask['completedTasks']
messages += remoteTask['messages']
except Exception, err:
log.err(err)
def _monitorPipeline(batchState):
"""
Monitors the current pipeline, propogating its children state to it
"""
pl = yield pipelines_client.pipelineList('localhost', 'local', 'guest',
batchState['pipeline_name'], True)
pl = pl[0]
numTasks = 6
completedTasks = 4
for cl, pName in pl['children']:
try:
_log(batchState, 'Loading child pipeline: (%s, %s)' % (cl, pName))
remotePipelines = yield pipelines_client.pipelineList(
'localhost', cl, 'guest', pName, True)
remotePipeline = remotePipelines[0]
_log(
batchState, 'Loading task for child pipeline: %s' %
remotePipeline['task_name'])
remoteTask = yield tasks_client.loadTask(
'localhost', cl, 'guest', remotePipeline['task_name'])
numTasks += remoteTask['numTasks']
completedTasks += remoteTask['completedTasks']
except Exception, err:
_log(batchState, 'Error in monitorPipeline: %s' % str(err))
def _monitorPipeline(batchState):
"""
Monitors the current pipeline, propogating its children state to it
"""
pl = yield pipelines_client.pipelineList('localhost',
'local',
'guest',
batchState['pipeline_name'],
True)
pl = pl[0]
numTasks = 9
completedTasks = 6
for cl, pName in pl['children']:
try:
remotePipelines = yield pipelines_client.pipelineList('localhost',
cl,
'guest',
pName,
True)
remotePipeline = remotePipelines[0]
_log(batchState, 'Loading child pipeline: (%s, %s, %s)' % (cl, pName, remotePipeline['task_name']))
remoteTask = yield tasks_client.loadTask('localhost',
cl,
'guest',
remotePipeline['task_name'])
numTasks += remoteTask['numTasks']
completedTasks += remoteTask['completedTasks']
except Exception, err:
_log(batchState, 'Error in monitorPipeline: %s' % str(err))
def blockOnTask(host, cluster, taskName, notifyF=logging.logPrint, errorF=logging.errorPrint):
endStates = [task.TASK_FAILED, task.TASK_COMPLETED]
state = None
prevTime = None
sleepTime = 1
yield defer_utils.sleep(sleepTime)()
while state not in endStates:
tsk = yield tasks_client.loadTask(host, cluster, 'guest', taskName)
tsk = task.taskFromDict(tsk)
state = tsk.state
if prevTime is None:
msgs = tsk.getMessages()
else:
msgs = tsk.getMessagesAfterTime(prevTime)
prevTime = tsk.timestamp
for m in msgs:
if m['mtype'] == task.MSG_ERROR:
errorF(m['text'])
elif m['mtype'] == task.MSG_NOTIFICATION:
notifyF(m['text'])
elif logging.DEBUG and m['mtype'] == task.MSG_SILENT:
logging.debugPrint(lambda : m['text'])
##
# Make this configurable
if state not in endStates:
sleepTime = sleepTime < 30 and sleepTime * 2 or 30
yield defer_utils.sleep(sleepTime)()
defer.returnValue(state)
def blockOnTask(host,
cluster,
taskName,
notifyF=logging.logPrint,
errorF=logging.errorPrint):
endStates = [task.TASK_FAILED, task.TASK_COMPLETED]
state = None
prevTime = None
sleepTime = 1
yield defer_utils.sleep(sleepTime)()
while state not in endStates:
tsk = yield tasks_client.loadTask(host, cluster, 'guest', taskName)
tsk = task.taskFromDict(tsk)
state = tsk.state
if prevTime is None:
msgs = tsk.getMessages()
else:
msgs = tsk.getMessagesAfterTime(prevTime)
prevTime = tsk.timestamp
for m in msgs:
if m['mtype'] == task.MSG_ERROR:
errorF(m['text'])
elif m['mtype'] == task.MSG_NOTIFICATION:
notifyF(m['text'])
elif logging.DEBUG and m['mtype'] == task.MSG_SILENT:
logging.debugPrint(lambda: m['text'])
##
# Make this configurable
if state not in endStates:
sleepTime = sleepTime < 30 and sleepTime * 2 or 30
yield defer_utils.sleep(sleepTime)()
defer.returnValue(state)
def _sumChildrenPipelines(self, pl):
numSteps = 0
completedSteps = 0
messages = []
clusters = yield clusters_client.listClusters('localhost',
{},
pl.userName)
clusterNames = set([c['cluster_name']
for c in clusters])
for cl, remotePipelineName in pl.children:
try:
if cl in clusterNames:
remotePipelines = yield pipelines_www.pipelineList('localhost',
cl,
pl.userName,
remotePipelineName,
True)
remotePipeline = remotePipelines[0]
remoteTask = yield tasks_www.loadTask('localhost',
cl,
pl.userName,
remotePipeline['task_name'])
numSteps += remoteTask['numTasks']
completedSteps += remoteTask['completedTasks']
messages += remoteTask['messages']
except Exception, err:
log.err(err)
def _waitForPipeline(batchState):
while True:
task = yield tasks_client.loadTask(
'localhost', batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest', batchState['pipeline_task'])
if task['state'] == tasks.task.TASK_COMPLETED:
break
elif task['state'] == tasks.task.TASK_FAILED:
raise Exception('Task failed - %s' % batchState['pipeline_task'])
yield defer_utils.sleep(30)()
def _waitForPipeline(batchState):
while True:
task = yield tasks_client.loadTask('localhost',
batchState['pipeline_config']['cluster.CLUSTER_NAME'],
'guest',
batchState['pipeline_task'])
if task['state'] == tasks.task.TASK_COMPLETED:
break
elif task['state'] == tasks.task.TASK_FAILED:
raise Exception('Task failed - %s' % batchState['pipeline_task'])
yield defer_utils.sleep(30)()
