def update_hash(self, idx, hashLocal = None):
file_hash = self._files[idx][0]
if hashLocal:
if file_hash == hashLocal:
result = Console.fmt('MATCH', [Console.COLOR_GREEN])
else:
result = Console.fmt('FAIL', [Console.COLOR_RED])
msg = '(R:%s L:%s) => %s' % (file_hash, hashLocal, result)
else:
msg = ''
self.output[2*idx] = self._infoline(idx, '(%s)' % self.tr[idx])
self.output[2*idx+1] = msg
def update_hash(self, idx, hashLocal=None):
file_hash = self._files[idx][0]
if hashLocal:
if file_hash == hashLocal:
result = Console.fmt('MATCH', [Console.COLOR_GREEN])
else:
result = Console.fmt('FAIL', [Console.COLOR_RED])
msg = '(R:%s L:%s) => %s' % (file_hash, hashLocal, result)
else:
msg = ''
self.output[2 * idx] = self._infoline(idx, '(%s)' % self.tr[idx])
self.output[2 * idx + 1] = msg
def display(self):
(catStateDict, catDescDict,
catSubcatDict) = self._getCategoryStateSummary()
self._catCur = len(catStateDict)
def sumCat(catKey, states):
return sum(imap(lambda z: catStateDict[catKey].get(z, 0), states))
self.printGUIHeader('Status report for task:')
for catKey in catStateDict: # sorted(catStateDict, key = lambda x: -self._categories[x][0]):
desc = self._formatDesc(catDescDict[catKey],
catSubcatDict.get(catKey, 0))
completed = sumCat(catKey, [Job.SUCCESS])
total = sum(catStateDict[catKey].values())
self.printLimited(
Console.fmt(desc, [Console.BOLD]), self.maxX - 24,
'(%5d jobs, %6.2f%% )' %
(total, 100 * completed / float(total)))
progressbar = JobProgressBar(sum(catStateDict[catKey].values()),
width=max(0, self.maxX - 19))
progressbar.update(
completed,
sumCat(catKey,
[Job.SUBMITTED, Job.WAITING, Job.READY, Job.QUEUED]),
sumCat(catKey, [Job.RUNNING, Job.DONE]),
sumCat(catKey, [Job.ABORTED, Job.CANCELLED, Job.FAILED]))
self.printLimited(progressbar, self.maxX)
for dummy in irange(self._catMax - len(catStateDict)):
sys.stdout.write(' ' * self.maxX + '\n')
sys.stdout.write(' ' * self.maxX + '\n')
def show_report(self, job_db, jobnum_list):
self._max_x = Console.getmaxyx()[1]
(cat_state_dict, map_cat2desc,
cat_subcat_dict) = self._cat_manager.get_category_infos(
job_db, jobnum_list)
self._cat_cur = len(cat_state_dict)
def _sum_cat(cat_key, states):
return sum(
imap(lambda z: cat_state_dict[cat_key].get(z, 0), states))
for cat_key in cat_state_dict: # Sorted(cat_state_dict, key=lambda x: -self._categories[x][0]):
desc = self._cat_manager.format_desc(
map_cat2desc[cat_key], cat_subcat_dict.get(cat_key, 0))
completed = _sum_cat(cat_key, [Job.SUCCESS])
total = sum(cat_state_dict[cat_key].values())
job_info_str = ''
if self._max_x > 65 + 23:
job_info_str = ' (%5d jobs, %6.2f%% )' % (
total, 100 * completed / float(total))
width = min(65 + len(job_info_str), self._max_x)
if len(desc) + len(job_info_str) > width:
desc = desc[:width - 3 - len(job_info_str)] + '...'
self._show_line(ANSI.bold + desc + ANSI.reset + ' ' *
(width -
(len(desc) + len(job_info_str))) + job_info_str)
progressbar = JobProgressBar(sum(cat_state_dict[cat_key].values()),
width=width,
display_text=(job_info_str != ''))
progressbar.update(
completed, _sum_cat(cat_key, JobClass.ATWMS.state_list),
_sum_cat(cat_key, [Job.RUNNING]),
_sum_cat(cat_key, [Job.DONE]),
_sum_cat(cat_key, [Job.ABORTED, Job.CANCELLED, Job.FAILED]))
self._show_line(str(progressbar))
def __init__(self, config, name, job_db, task=None):
ConsoleReport.__init__(self, config, name, job_db, task)
(max_y, self._max_x) = Console.getmaxyx()
cat_max = config.get_int('report categories max',
int(max_y / 5),
on_change=None)
self._cat_manager = AdaptiveJobCategoryManager(config, job_db, task,
cat_max)
self._cat_cur = cat_max
def displayWorkflow(self):
if not sys.stdout.isatty():
return self._workflow.process(self._wait)
self._console = Console(sys.stdout)
self._new_stdout = GUIStream(sys.stdout, self._console, self._lock)
self._new_stderr = GUIStream(sys.stderr, self._console, self._lock)
Activity.callbacks.append(self._schedule_update_report_status)
try:
# Main cycle - GUI mode
(sys.stdout, sys.stderr) = (self._new_stdout, self._new_stderr)
self._console.erase()
self._schedule_update_layout()
self._workflow.process(self._wait)
finally:
(sys.stdout, sys.stderr) = (self._stored_stdout,
self._stored_stderr)
self._console.setscrreg()
self._console.erase()
self._update_all()
def _write(self, value):
max_yx = Console.getmaxyx()
max_x = min(max_yx[1], self._msg_len_max)
value = value.replace('\n', ANSI.erase_line + '\n' + ANSI.erase_line)
self._stream.write(value + '\n' + ANSI.wrap_off)
activity_list = list(Activity.root.get_children())
max_depth = int(max_yx[0] * self._fold)
while len(activity_list) > int(max_yx[0] * self._fold):
activity_list = lfilter(
lambda activity: activity.depth - 1 < max_depth, activity_list)
max_depth -= 1
for activity in activity_list:
msg = self._format_activity(max_x, activity.depth - 1, activity,
activity_list)
self._stream.write(ANSI.erase_line + msg + '\n')
self._stream.write(ANSI.erase_down +
ANSI.move_up(len(activity_list) + 1) + ANSI.wrap_on)
self._stream.flush()
def displayWorkflow(self): if not sys.stdout.isatty(): return self._workflow.process(self._wait) self._console = Console(sys.stdout) self._new_stdout = GUIStream(sys.stdout, self._console, self._lock) self._new_stderr = GUIStream(sys.stderr, self._console, self._lock) Activity.callbacks.append(self._schedule_update_report_status) try: # Main cycle - GUI mode (sys.stdout, sys.stderr) = (self._new_stdout, self._new_stderr) self._console.erase() self._schedule_update_layout() self._workflow.process(self._wait) finally: (sys.stdout, sys.stderr) = (self._stored_stdout, self._stored_stderr) self._console.setscrreg() self._console.erase() self._update_all()
def display(self):
(catStateDict, catDescDict, catSubcatDict) = self._getCategoryStateSummary()
self._catCur = len(catStateDict)
sumCat = lambda catKey, states: sum(map(lambda z: catStateDict[catKey].get(z, 0), states))
self.printGUIHeader('Status report for task:')
for catKey in catStateDict: #sorted(catStateDict, key = lambda x: -self._categories[x][0]):
desc = self._formatDesc(catDescDict[catKey], catSubcatDict.get(catKey, 0))
completed = sumCat(catKey, [Job.SUCCESS])
total = sum(catStateDict[catKey].values())
self.printLimited(Console.fmt(desc, [Console.BOLD]), self.maxX - 24,
'(%5d jobs, %6.2f%% )' % (total, 100 * completed / float(total)))
bar = JobProgressBar(sum(catStateDict[catKey].values()), width = self.maxX - 19)
bar.update(completed,
sumCat(catKey, [Job.SUBMITTED, Job.WAITING, Job.READY, Job.QUEUED]),
sumCat(catKey, [Job.RUNNING, Job.DONE]),
sumCat(catKey, [Job.ABORTED, Job.CANCELLED, Job.FAILED]))
self.printLimited(bar, self.maxX)
for x in range(self._catMax - len(catStateDict)):
print ' ' * self.maxX
print ' ' * self.maxX
def display(self):
(catStateDict, catDescDict, catSubcatDict) = self._getCategoryStateSummary()
self._catCur = len(catStateDict)
def sumCat(catKey, states):
return sum(imap(lambda z: catStateDict[catKey].get(z, 0), states))
self.printGUIHeader('Status report for task:')
for catKey in catStateDict: # sorted(catStateDict, key = lambda x: -self._categories[x][0]):
desc = self._formatDesc(catDescDict[catKey], catSubcatDict.get(catKey, 0))
completed = sumCat(catKey, [Job.SUCCESS])
total = sum(catStateDict[catKey].values())
self.printLimited(Console.fmt(desc, [Console.BOLD]), self.maxX - 24,
'(%5d jobs, %6.2f%% )' % (total, 100 * completed / float(total)))
progressbar = JobProgressBar(sum(catStateDict[catKey].values()), width = max(0, self.maxX - 19))
progressbar.update(completed,
sumCat(catKey, JobClass.ATWMS.states),
sumCat(catKey, [Job.RUNNING, Job.DONE]),
sumCat(catKey, [Job.ABORTED, Job.CANCELLED, Job.FAILED]))
self.printLimited(progressbar, self.maxX)
for dummy in irange(self._catMax - len(catStateDict)):
sys.stdout.write(' ' * self.maxX + '\n')
sys.stdout.write(' ' * self.maxX + '\n')
class ANSIGUI(GUI):
def __init__(self, config, workflow):
config.set('report', 'BasicReport BarReport')
(self._stored_stdout, self._stored_stderr) = (sys.stdout, sys.stderr)
GUI.__init__(self, config, workflow)
self._reportHeight = 0
self._statusHeight = 1
self._old_message = None
self._lock = GCLock(threading.RLock()) # drawing lock
self._last_report = 0
self._old_size = None
def _draw(self, fun):
new_size = self._console.getmaxyx()
if self._old_size != new_size:
self._old_size = new_size
self._schedule_update_layout()
self._lock.acquire()
self._console.hideCursor()
self._console.savePos()
try:
fun()
finally:
self._console.loadPos()
self._console.showCursor()
self._lock.release()
# Event handling for resizing
def _update_layout(self):
(sizey, sizex) = self._console.getmaxyx()
self._old_size = (sizey, sizex)
self._reportHeight = self._report.getHeight()
self._console.erase()
self._console.setscrreg(
min(self._reportHeight + self._statusHeight + 1, sizey), sizey)
utils.printTabular.wraplen = sizex - 5
self._update_all()
def _schedule_update_layout(self, sig=None, frame=None):
start_thread(
'update layout', self._draw,
self._update_layout) # using new thread to ensure RLock is free
def _wait(self, timeout):
oldHandler = signal.signal(signal.SIGWINCH,
self._schedule_update_layout)
result = utils.wait(timeout)
signal.signal(signal.SIGWINCH, oldHandler)
return result
def _update_report(self):
if time.time() - self._last_report < 1:
return
self._last_report = time.time()
self._console.move(0, 0)
self._new_stdout.logged = False
self._report.display()
self._new_stdout.logged = True
def _update_status(self):
activity_message = None
for activity in Activity.root.get_children():
activity_message = activity.getMessage() + '...'
if len(activity_message) > 75:
activity_message = activity_message[:
37] + '...' + activity_message[
-35:]
self._console.move(self._reportHeight + 1, 0)
self._new_stdout.logged = False
if self._old_message:
self._stored_stdout.write(self._old_message.center(65) + '\r')
self._stored_stdout.flush()
self._old_message = activity_message
if activity_message:
self._stored_stdout.write('%s' % activity_message.center(65))
self._stored_stdout.flush()
self._new_stdout.logged = True
def _update_log(self):
self._console.move(self._reportHeight + 2, 0)
self._console.eraseDown()
self._new_stdout.dump()
def _update_all(self):
self._last_report = 0
self._update_report()
self._update_status()
self._update_log()
def _schedule_update_report_status(self):
self._draw(self._update_report)
self._draw(self._update_status)
def displayWorkflow(self):
if not sys.stdout.isatty():
return self._workflow.process(self._wait)
self._console = Console(sys.stdout)
self._new_stdout = GUIStream(sys.stdout, self._console, self._lock)
self._new_stderr = GUIStream(sys.stderr, self._console, self._lock)
Activity.callbacks.append(self._schedule_update_report_status)
try:
# Main cycle - GUI mode
(sys.stdout, sys.stderr) = (self._new_stdout, self._new_stderr)
self._console.erase()
self._schedule_update_layout()
self._workflow.process(self._wait)
finally:
(sys.stdout, sys.stderr) = (self._stored_stdout,
self._stored_stderr)
self._console.setscrreg()
self._console.erase()
self._update_all()
def update(self, success = 0, queued = 0, running = 0, failed = 0):
# Compute variables
complete = self._width - 31
blocks_ok = int(round(success / float(self._total) * complete))
blocks_proc = int(round(queued / float(self._total) * complete))
blocks_run = int(round(running / float(self._total) * complete))
blocks_fail = min(complete - blocks_ok - blocks_proc - blocks_run,
int(round(failed / float(self._total) * complete)))
self._bar = str(Console.fmt('=' * blocks_ok, [Console.COLOR_GREEN]))
self._bar += str(Console.fmt('=' * blocks_run, [Console.COLOR_BLUE]))
self._bar += str(Console.fmt('=' * blocks_proc, [Console.COLOR_WHITE]))
self._bar += ' ' * (complete - blocks_ok - blocks_proc - blocks_run - blocks_fail)
self._bar += str(Console.fmt('=' * blocks_fail, [Console.COLOR_RED]))
self._bar = '[%s] ' % self._bar
if success == self._total and self._jobsOnFinish:
self._bar += '(%s |%s)' % (Console.fmt('%5d' % self._total, [Console.COLOR_GREEN]),
Console.fmt('finished'.center(14), [Console.COLOR_GREEN]))
elif success == self._total:
self._bar += '(%s)' % (Console.fmt('finished'.center(21), [Console.COLOR_GREEN]))
else:
fmt = lambda x: str(x).rjust(5)#int(math.log(self._total) / math.log(10)) + 1)
self._bar += '(%s | %s | %s | %s)' % (
Console.fmt(fmt(success), [Console.COLOR_GREEN]),
Console.fmt(fmt(running), [Console.COLOR_BLUE]),
Console.fmt(fmt(queued), [Console.COLOR_WHITE]),
Console.fmt(fmt(failed), [Console.COLOR_RED]))
class ANSIGUI(GUI):
def __init__(self, config, workflow):
config.set('report', 'BasicReport BarReport')
(self._stored_stdout, self._stored_stderr) = (sys.stdout, sys.stderr)
GUI.__init__(self, config, workflow)
self._reportHeight = 0
self._statusHeight = 1
self._old_message = None
self._lock = GCLock(threading.RLock()) # drawing lock
self._last_report = 0
self._old_size = None
def _draw(self, fun):
new_size = self._console.getmaxyx()
if self._old_size != new_size:
self._old_size = new_size
self._schedule_update_layout()
self._lock.acquire()
self._console.hideCursor()
self._console.savePos()
try:
fun()
finally:
self._console.loadPos()
self._console.showCursor()
self._lock.release()
# Event handling for resizing
def _update_layout(self):
(sizey, sizex) = self._console.getmaxyx()
self._old_size = (sizey, sizex)
self._reportHeight = self._report.getHeight()
self._console.erase()
self._console.setscrreg(min(self._reportHeight + self._statusHeight + 1, sizey), sizey)
utils.printTabular.wraplen = sizex - 5
self._update_all()
def _schedule_update_layout(self, sig = None, frame = None):
start_thread('update layout', self._draw, self._update_layout) # using new thread to ensure RLock is free
def _wait(self, timeout):
oldHandler = signal.signal(signal.SIGWINCH, self._schedule_update_layout)
result = utils.wait(timeout)
signal.signal(signal.SIGWINCH, oldHandler)
return result
def _update_report(self):
if time.time() - self._last_report < 1:
return
self._last_report = time.time()
self._console.move(0, 0)
self._new_stdout.logged = False
self._report.display()
self._new_stdout.logged = True
def _update_status(self):
activity_message = None
for activity in Activity.root.get_children():
activity_message = activity.getMessage() + '...'
if len(activity_message) > 75:
activity_message = activity_message[:37] + '...' + activity_message[-35:]
self._console.move(self._reportHeight + 1, 0)
self._new_stdout.logged = False
if self._old_message:
self._stored_stdout.write(self._old_message.center(65) + '\r')
self._stored_stdout.flush()
self._old_message = activity_message
if activity_message:
self._stored_stdout.write('%s' % activity_message.center(65))
self._stored_stdout.flush()
self._new_stdout.logged = True
def _update_log(self):
self._console.move(self._reportHeight + 2, 0)
self._console.eraseDown()
self._new_stdout.dump()
def _update_all(self):
self._last_report = 0
self._update_report()
self._update_status()
self._update_log()
def _schedule_update_report_status(self):
self._draw(self._update_report)
self._draw(self._update_status)
def displayWorkflow(self):
if not sys.stdout.isatty():
return self._workflow.process(self._wait)
self._console = Console(sys.stdout)
self._new_stdout = GUIStream(sys.stdout, self._console, self._lock)
self._new_stderr = GUIStream(sys.stderr, self._console, self._lock)
Activity.callbacks.append(self._schedule_update_report_status)
try:
# Main cycle - GUI mode
(sys.stdout, sys.stderr) = (self._new_stdout, self._new_stderr)
self._console.erase()
self._schedule_update_layout()
self._workflow.process(self._wait)
finally:
(sys.stdout, sys.stderr) = (self._stored_stdout, self._stored_stderr)
self._console.setscrreg()
self._console.erase()
self._update_all()
def download_multithreaded(opts, workDir, jobList, incInfo, jobDB, token):
from grid_control_gui.ansi import Console
errorOutput = []
class ThreadDisplay:
def __init__(self):
self.output = []
def init(self, jobNum):
self.jobNum = jobNum
self.output = ['Job %5d' % jobNum, '']
def _infoline(self, fileIdx, msg=''):
return 'Job %5d [%i/%i] %s %s' % (self.jobNum, fileIdx + 1,
len(self._files),
self._files[fileIdx][2], msg)
def update_files(self, files):
(self._files, self.output, self.tr) = (files, self.output[1:],
[''] * len(files))
for x in irange(len(files)):
self.output.insert(2 * x, self._infoline(x))
self.output.insert(2 * x + 1, '')
def update_progress(self,
idx,
csize=None,
osize=None,
stime=None,
otime=None):
if otime:
trfun = lambda sref, tref: gcSupport.prettySize(
((csize - sref) / max(1,
time.time() - tref)))
self.tr[idx] = '%7s avg. - %7s/s inst.' % (
gcSupport.prettySize(csize), trfun(0, stime))
self.output[2 * idx] = self._infoline(
idx, '(%s - %7s/s)' % (self.tr[idx], trfun(osize, otime)))
def update_hash(self, idx, hashLocal=None):
file_hash = self._files[idx][0]
if hashLocal:
if file_hash == hashLocal:
result = Console.fmt('MATCH', [Console.COLOR_GREEN])
else:
result = Console.fmt('FAIL', [Console.COLOR_RED])
msg = '(R:%s L:%s) => %s' % (file_hash, hashLocal, result)
else:
msg = ''
self.output[2 * idx] = self._infoline(idx, '(%s)' % self.tr[idx])
self.output[2 * idx + 1] = msg
def error(self, msg):
errorOutput.append(msg)
def update_status(self, idx, msg):
self.output[2 * idx] = str.join(
' ', [self._infoline(idx, '(%s)' % self.tr[idx])] +
(msg or '').split())
def finish(self):
# self.output.append(str(self.jobNum) + 'FINISHED')
pass
download_multithreaded_main(opts, workDir, jobList, incInfo, jobDB, token,
ThreadDisplay, Console(), errorOutput)
def __init__(self, jobDB, task, jobs=None, configString=''):
(self.maxY, self.maxX) = Console().getmaxyx()
self.maxX -= 10 # Padding
AdaptiveBaseReport.__init__(self, jobDB, task, jobs,
str(int(self.maxY / 5)))
def update(self, success=0, queued=0, running=0, failed=0):
# Compute variables
complete = self._width - 31
blocks_ok = int(round(success / float(self._total) * complete))
blocks_proc = int(round(queued / float(self._total) * complete))
blocks_run = int(round(running / float(self._total) * complete))
blocks_fail = min(complete - blocks_ok - blocks_proc - blocks_run,
int(round(failed / float(self._total) * complete)))
self._bar = str(Console.fmt('=' * blocks_ok, [Console.COLOR_GREEN]))
self._bar += str(Console.fmt('=' * blocks_run, [Console.COLOR_BLUE]))
self._bar += str(Console.fmt('=' * blocks_proc, [Console.COLOR_WHITE]))
self._bar += ' ' * (complete - blocks_ok - blocks_proc - blocks_run -
blocks_fail)
self._bar += str(Console.fmt('=' * blocks_fail, [Console.COLOR_RED]))
self._bar = '[%s] ' % self._bar
if success == self._total and self._jobsOnFinish:
self._bar += '(%s |%s)' % (Console.fmt('%5d' % self._total, [
Console.COLOR_GREEN
]), Console.fmt('finished'.center(14), [Console.COLOR_GREEN]))
elif success == self._total:
self._bar += '(%s)' % (Console.fmt('finished'.center(21),
[Console.COLOR_GREEN]))
else:
fmt = lambda x: str(x).rjust(
5) # int(math.log(self._total) / math.log(10)) + 1)
self._bar += '(%s | %s | %s | %s)' % (
Console.fmt(fmt(success), [Console.COLOR_GREEN]),
Console.fmt(fmt(running), [Console.COLOR_BLUE]),
Console.fmt(fmt(queued), [Console.COLOR_WHITE]),
Console.fmt(fmt(failed), [Console.COLOR_RED]))
def displayWorkflow(self):
gui = self
report = self._report
workflow = self._workflow
def wrapper(screen):
# Event handling for resizing
def onResize(sig, frame):
self._reportHeight = report.getHeight()
screen.savePos()
(sizey, sizex) = screen.getmaxyx()
screen.setscrreg(min(self.getHeight() + 2, sizey), sizey)
utils.printTabular.wraplen = sizex - 5
screen.loadPos()
screen.erase()
onResize(None, None)
def guiWait(timeout):
onResize(None, None)
oldHandler = signal.signal(signal.SIGWINCH, onResize)
result = utils.wait(timeout)
signal.signal(signal.SIGWINCH, oldHandler)
if (time.time() - guiWait.lastwait > 10) and not timeout:
tmp = utils.ActivityLog('') # force display update
tmp.finish()
guiWait.lastwait = time.time()
return result
guiWait.lastwait = 0
# Wrapping ActivityLog functionality
class GUILog(object):
def __init__(self, message):
self._message = '%s...' % message
self.show(self._message)
def finish(self):
if hasattr(sys.stdout, 'logged') and self._message:
self.show(' ' * len(self._message))
self._message = []
def __del__(self):
self.finish()
def show(self, message):
if gui.getHeight() != report.getHeight():
screen.erase()
onResize(None, None)
sys.stdout.dump()
screen.savePos()
screen.move(0, 0)
sys.stdout.logged = False
report.display()
screen.move(gui.getHeight() + 1, 0)
sys.stdout.write('%s\n' % message.center(65))
sys.stdout.logged = True
screen.loadPos()
# Main cycle - GUI mode
saved = (sys.stdout, sys.stderr, utils.ActivityLog)
try:
utils.ActivityLog = GUILog
sys.stdout = GUIStream(saved[0], screen)
sys.stderr = GUIStream(saved[1], screen)
workflow.jobCycle(guiWait)
finally:
sys.stdout, sys.stderr, utils.ActivityLog = saved
screen.setscrreg()
screen.move(1, 0)
screen.eraseDown()
report.display()
sys.stdout.write('\n')
try:
wrapper(Console())
finally:
GUIStream.dump()
def _end_interface():
try:
self._finish_drawing()
finally:
GCStreamHandler.set_global_lock()
Console.reset_console()
