def download_monitored(jobNum, output, fileIdx, checkPath, sourcePath,
targetPath):
copyAbortLock = GCLock()
monitorLock = GCLock()
monitorLock.acquire()
monitor = start_thread('Download monitor %s' % jobNum, transfer_monitor,
output, fileIdx, checkPath, monitorLock,
copyAbortLock)
result = -1
procCP = se_copy(sourcePath, targetPath, tmp=checkPath)
while True:
if not copyAbortLock.acquire(False):
monitor.join()
break
copyAbortLock.release()
result = procCP.status(timeout=0)
if result is not None:
monitorLock.release()
monitor.join()
break
time.sleep(0.02)
if result != 0:
output.error('Unable to copy file from SE!')
log.critical('%s\n%s\n', procCP.stdout.read(timeout=0),
procCP.stderr.read(timeout=0))
return False
return True
class BaseJobFileTarAdaptor(object):
def __init__(self, path):
activity = Activity('Reading dataset partition file')
self._lock = GCLock()
self._fmt = utils.DictFormat()
self._tar = tarfile.open(path, 'r:')
(self._cacheKey, self._cacheTar) = (None, None)
metadata = self._fmt.parse(self._tar.extractfile('Metadata').readlines(), keyParser = {None: str})
self.maxJobs = metadata.pop('MaxJobs')
self.classname = metadata.pop('ClassName')
self.metadata = {'dataset': dict(ifilter(lambda k_v: not k_v[0].startswith('['), metadata.items()))}
for (k, v) in ifilter(lambda k_v: k_v[0].startswith('['), metadata.items()):
self.metadata.setdefault('dataset %s' % k.split(']')[0].lstrip('['), {})[k.split(']')[1].strip()] = v
activity.finish()
self._parserMap = { None: str, DataSplitter.NEntries: int, DataSplitter.Skipped: int,
DataSplitter.DatasetID: int, DataSplitter.Invalid: parseBool,
DataSplitter.Locations: lambda x: parseList(x, ','),
DataSplitter.MetadataHeader: parseJSON,
DataSplitter.Metadata: lambda x: parseJSON(x.strip("'")) }
def __getitem__(self, key):
if key >= self.maxJobs:
raise IndexError('Invalid dataset partition %s' % repr(key))
try:
self._lock.acquire()
return self._getPartition(key)
finally:
self._lock.release()
class StatusMonitor(object):
def __init__(self, num_jobs):
self._result = {}
self._lock = GCLock()
self._num_jobs = num_jobs
def is_finished(self):
num_success = sum(
imap(lambda jds: self._result.get(jds, 0), [
JobDownloadStatus.JOB_OK, JobDownloadStatus.JOB_ALREADY,
JobDownloadStatus.JOB_INCOMPLETE
]))
return num_success == self._num_jobs
def register_file_result(self, jobnum, fi_idx, msg, status, proc=None):
if proc:
log.log_process(proc, msg=log_intro(jobnum, fi_idx) + msg)
else:
log.info(log_intro(jobnum, fi_idx) + msg)
self._register_result(status)
return status # returned file status is actually used later
def register_job_result(self, jobnum, msg, status):
log.info(log_intro(jobnum) + msg)
self._register_result(status)
def show_results(self):
def _iter_download_results(cls):
marker = False
for stat in sorted(self._result, key=self._result.get):
if self._result[stat] and (stat in cls.enum_value_list):
yield {0: cls.enum2str(stat), 1: self._result[stat]}
marker = True
if marker:
yield '='
yield {0: 'Total', 1: self._num_jobs}
if self._result:
ConsoleTable.create([(0, 'Status'), (1, '')],
_iter_download_results(JobDownloadStatus),
title='Job status overview')
ConsoleTable.create([(0, 'Status'), (1, '')],
_iter_download_results(FileDownloadStatus),
title='File status overview')
def _register_result(self, status):
self._lock.acquire()
try:
self._result[status] = self._result.get(status, 0) + 1
finally:
self._lock.release()
class GCStreamHandler(logging.Handler): def __init__(self): logging.Handler.__init__(self) self._lock = GCLock(threading.RLock()) def get_stream(self): raise AbstractError def emit(self, record): self._lock.acquire() try: stream = self.get_stream() stream.write(self.format(record) + '\n') stream.flush() finally: self._lock.release()
class GCStreamHandler(logging.Handler):
def __init__(self):
logging.Handler.__init__(self)
self._lock = GCLock(threading.RLock())
def get_stream(self):
raise AbstractError
def emit(self, record):
self._lock.acquire()
try:
stream = self.get_stream()
stream.write(self.format(record) + '\n')
stream.flush()
finally:
self._lock.release()
class StatusMonitor(object):
def __init__(self, num_jobs):
self._result = {}
self._lock = GCLock()
self._num_jobs = num_jobs
def is_finished(self):
num_success = sum(imap(lambda jds: self._result.get(jds, 0), [
JobDownloadStatus.JOB_OK, JobDownloadStatus.JOB_ALREADY, JobDownloadStatus.JOB_INCOMPLETE]))
return num_success == self._num_jobs
def register_file_result(self, jobnum, fi_idx, msg, status, proc=None):
if proc:
log.log_process(proc, msg=log_intro(jobnum, fi_idx) + msg)
else:
log.info(log_intro(jobnum, fi_idx) + msg)
self._register_result(status)
return status # returned file status is actually used later
def register_job_result(self, jobnum, msg, status):
log.info(log_intro(jobnum) + msg)
self._register_result(status)
def show_results(self):
def _iter_download_results(cls):
marker = False
for stat in sorted(self._result, key=self._result.get):
if self._result[stat] and (stat in cls.enum_value_list):
yield {0: cls.enum2str(stat), 1: self._result[stat]}
marker = True
if marker:
yield '='
yield {0: 'Total', 1: self._num_jobs}
if self._result:
ConsoleTable.create([(0, 'Status'), (1, '')],
_iter_download_results(JobDownloadStatus), title='Job status overview')
ConsoleTable.create([(0, 'Status'), (1, '')],
_iter_download_results(FileDownloadStatus), title='File status overview')
def _register_result(self, status):
self._lock.acquire()
try:
self._result[status] = self._result.get(status, 0) + 1
finally:
self._lock.release()
def download_monitored(jobNum, output, fileIdx, checkPath, sourcePath, targetPath):
copyAbortLock = GCLock()
monitorLock = GCLock()
monitorLock.acquire()
monitor = start_thread('Download monitor %s' % jobNum, transfer_monitor, output, fileIdx, checkPath, monitorLock, copyAbortLock)
result = -1
procCP = se_copy(sourcePath, targetPath, tmp = checkPath)
while True:
if not copyAbortLock.acquire(False):
monitor.join()
break
copyAbortLock.release()
result = procCP.status(timeout = 0)
if result is not None:
monitorLock.release()
monitor.join()
break
time.sleep(0.02)
if result != 0:
output.error('Unable to copy file from SE!')
log.critical('%s\n%s\n', procCP.stdout.read(timeout = 0), procCP.stderr.read(timeout = 0))
return False
return True
class ANSIGUI(GUI):
def __init__(self, config, workflow):
config.set('report', 'BasicReport BarReport')
(self._stored_stdout, self._stored_stderr) = (sys.stdout, sys.stderr)
GUI.__init__(self, config, workflow)
self._reportHeight = 0
self._statusHeight = 1
self._old_message = None
self._lock = GCLock(threading.RLock()) # drawing lock
self._last_report = 0
self._old_size = None
def _draw(self, fun):
new_size = self._console.getmaxyx()
if self._old_size != new_size:
self._old_size = new_size
self._schedule_update_layout()
self._lock.acquire()
self._console.hideCursor()
self._console.savePos()
try:
fun()
finally:
self._console.loadPos()
self._console.showCursor()
self._lock.release()
# Event handling for resizing
def _update_layout(self):
(sizey, sizex) = self._console.getmaxyx()
self._old_size = (sizey, sizex)
self._reportHeight = self._report.getHeight()
self._console.erase()
self._console.setscrreg(
min(self._reportHeight + self._statusHeight + 1, sizey), sizey)
utils.printTabular.wraplen = sizex - 5
self._update_all()
def _schedule_update_layout(self, sig=None, frame=None):
start_thread(
'update layout', self._draw,
self._update_layout) # using new thread to ensure RLock is free
def _wait(self, timeout):
oldHandler = signal.signal(signal.SIGWINCH,
self._schedule_update_layout)
result = utils.wait(timeout)
signal.signal(signal.SIGWINCH, oldHandler)
return result
def _update_report(self):
if time.time() - self._last_report < 1:
return
self._last_report = time.time()
self._console.move(0, 0)
self._new_stdout.logged = False
self._report.display()
self._new_stdout.logged = True
def _update_status(self):
activity_message = None
for activity in Activity.root.get_children():
activity_message = activity.getMessage() + '...'
if len(activity_message) > 75:
activity_message = activity_message[:
37] + '...' + activity_message[
-35:]
self._console.move(self._reportHeight + 1, 0)
self._new_stdout.logged = False
if self._old_message:
self._stored_stdout.write(self._old_message.center(65) + '\r')
self._stored_stdout.flush()
self._old_message = activity_message
if activity_message:
self._stored_stdout.write('%s' % activity_message.center(65))
self._stored_stdout.flush()
self._new_stdout.logged = True
def _update_log(self):
self._console.move(self._reportHeight + 2, 0)
self._console.eraseDown()
self._new_stdout.dump()
def _update_all(self):
self._last_report = 0
self._update_report()
self._update_status()
self._update_log()
def _schedule_update_report_status(self):
self._draw(self._update_report)
self._draw(self._update_status)
def displayWorkflow(self):
if not sys.stdout.isatty():
return self._workflow.process(self._wait)
self._console = Console(sys.stdout)
self._new_stdout = GUIStream(sys.stdout, self._console, self._lock)
self._new_stderr = GUIStream(sys.stderr, self._console, self._lock)
Activity.callbacks.append(self._schedule_update_report_status)
try:
# Main cycle - GUI mode
(sys.stdout, sys.stderr) = (self._new_stdout, self._new_stderr)
self._console.erase()
self._schedule_update_layout()
self._workflow.process(self._wait)
finally:
(sys.stdout, sys.stderr) = (self._stored_stdout,
self._stored_stderr)
self._console.setscrreg()
self._console.erase()
self._update_all()
class ANSIGUI(GUI):
def __init__(self, config, workflow):
config.set('report', 'BasicReport BarReport')
(self._stored_stdout, self._stored_stderr) = (sys.stdout, sys.stderr)
GUI.__init__(self, config, workflow)
self._reportHeight = 0
self._statusHeight = 1
self._old_message = None
self._lock = GCLock(threading.RLock()) # drawing lock
self._last_report = 0
self._old_size = None
def _draw(self, fun):
new_size = self._console.getmaxyx()
if self._old_size != new_size:
self._old_size = new_size
self._schedule_update_layout()
self._lock.acquire()
self._console.hideCursor()
self._console.savePos()
try:
fun()
finally:
self._console.loadPos()
self._console.showCursor()
self._lock.release()
# Event handling for resizing
def _update_layout(self):
(sizey, sizex) = self._console.getmaxyx()
self._old_size = (sizey, sizex)
self._reportHeight = self._report.getHeight()
self._console.erase()
self._console.setscrreg(min(self._reportHeight + self._statusHeight + 1, sizey), sizey)
utils.printTabular.wraplen = sizex - 5
self._update_all()
def _schedule_update_layout(self, sig = None, frame = None):
start_thread('update layout', self._draw, self._update_layout) # using new thread to ensure RLock is free
def _wait(self, timeout):
oldHandler = signal.signal(signal.SIGWINCH, self._schedule_update_layout)
result = utils.wait(timeout)
signal.signal(signal.SIGWINCH, oldHandler)
return result
def _update_report(self):
if time.time() - self._last_report < 1:
return
self._last_report = time.time()
self._console.move(0, 0)
self._new_stdout.logged = False
self._report.display()
self._new_stdout.logged = True
def _update_status(self):
activity_message = None
for activity in Activity.root.get_children():
activity_message = activity.getMessage() + '...'
if len(activity_message) > 75:
activity_message = activity_message[:37] + '...' + activity_message[-35:]
self._console.move(self._reportHeight + 1, 0)
self._new_stdout.logged = False
if self._old_message:
self._stored_stdout.write(self._old_message.center(65) + '\r')
self._stored_stdout.flush()
self._old_message = activity_message
if activity_message:
self._stored_stdout.write('%s' % activity_message.center(65))
self._stored_stdout.flush()
self._new_stdout.logged = True
def _update_log(self):
self._console.move(self._reportHeight + 2, 0)
self._console.eraseDown()
self._new_stdout.dump()
def _update_all(self):
self._last_report = 0
self._update_report()
self._update_status()
self._update_log()
def _schedule_update_report_status(self):
self._draw(self._update_report)
self._draw(self._update_status)
def displayWorkflow(self):
if not sys.stdout.isatty():
return self._workflow.process(self._wait)
self._console = Console(sys.stdout)
self._new_stdout = GUIStream(sys.stdout, self._console, self._lock)
self._new_stderr = GUIStream(sys.stderr, self._console, self._lock)
Activity.callbacks.append(self._schedule_update_report_status)
try:
# Main cycle - GUI mode
(sys.stdout, sys.stderr) = (self._new_stdout, self._new_stderr)
self._console.erase()
self._schedule_update_layout()
self._workflow.process(self._wait)
finally:
(sys.stdout, sys.stderr) = (self._stored_stdout, self._stored_stderr)
self._console.setscrreg()
self._console.erase()
self._update_all()