def download_monitored(jobNum, output, fileIdx, checkPath, sourcePath, targetPath): copyAbortLock = GCLock() monitorLock = GCLock() monitorLock.acquire() monitor = start_thread('Download monitor %s' % jobNum, transfer_monitor, output, fileIdx, checkPath, monitorLock, copyAbortLock) result = -1 procCP = se_copy(sourcePath, targetPath, tmp=checkPath) while True: if not copyAbortLock.acquire(False): monitor.join() break copyAbortLock.release() result = procCP.status(timeout=0) if result is not None: monitorLock.release() monitor.join() break time.sleep(0.02) if result != 0: output.error('Unable to copy file from SE!') log.critical('%s\n%s\n', procCP.stdout.read(timeout=0), procCP.stderr.read(timeout=0)) return False return True
class BaseJobFileTarAdaptor(object): def __init__(self, path): activity = Activity('Reading dataset partition file') self._lock = GCLock() self._fmt = utils.DictFormat() self._tar = tarfile.open(path, 'r:') (self._cacheKey, self._cacheTar) = (None, None) metadata = self._fmt.parse(self._tar.extractfile('Metadata').readlines(), keyParser = {None: str}) self.maxJobs = metadata.pop('MaxJobs') self.classname = metadata.pop('ClassName') self.metadata = {'dataset': dict(ifilter(lambda k_v: not k_v[0].startswith('['), metadata.items()))} for (k, v) in ifilter(lambda k_v: k_v[0].startswith('['), metadata.items()): self.metadata.setdefault('dataset %s' % k.split(']')[0].lstrip('['), {})[k.split(']')[1].strip()] = v activity.finish() self._parserMap = { None: str, DataSplitter.NEntries: int, DataSplitter.Skipped: int, DataSplitter.DatasetID: int, DataSplitter.Invalid: parseBool, DataSplitter.Locations: lambda x: parseList(x, ','), DataSplitter.MetadataHeader: parseJSON, DataSplitter.Metadata: lambda x: parseJSON(x.strip("'")) } def __getitem__(self, key): if key >= self.maxJobs: raise IndexError('Invalid dataset partition %s' % repr(key)) try: self._lock.acquire() return self._getPartition(key) finally: self._lock.release()
class StatusMonitor(object): def __init__(self, num_jobs): self._result = {} self._lock = GCLock() self._num_jobs = num_jobs def is_finished(self): num_success = sum( imap(lambda jds: self._result.get(jds, 0), [ JobDownloadStatus.JOB_OK, JobDownloadStatus.JOB_ALREADY, JobDownloadStatus.JOB_INCOMPLETE ])) return num_success == self._num_jobs def register_file_result(self, jobnum, fi_idx, msg, status, proc=None): if proc: log.log_process(proc, msg=log_intro(jobnum, fi_idx) + msg) else: log.info(log_intro(jobnum, fi_idx) + msg) self._register_result(status) return status # returned file status is actually used later def register_job_result(self, jobnum, msg, status): log.info(log_intro(jobnum) + msg) self._register_result(status) def show_results(self): def _iter_download_results(cls): marker = False for stat in sorted(self._result, key=self._result.get): if self._result[stat] and (stat in cls.enum_value_list): yield {0: cls.enum2str(stat), 1: self._result[stat]} marker = True if marker: yield '=' yield {0: 'Total', 1: self._num_jobs} if self._result: ConsoleTable.create([(0, 'Status'), (1, '')], _iter_download_results(JobDownloadStatus), title='Job status overview') ConsoleTable.create([(0, 'Status'), (1, '')], _iter_download_results(FileDownloadStatus), title='File status overview') def _register_result(self, status): self._lock.acquire() try: self._result[status] = self._result.get(status, 0) + 1 finally: self._lock.release()
class GCStreamHandler(logging.Handler): def __init__(self): logging.Handler.__init__(self) self._lock = GCLock(threading.RLock()) def get_stream(self): raise AbstractError def emit(self, record): self._lock.acquire() try: stream = self.get_stream() stream.write(self.format(record) + '\n') stream.flush() finally: self._lock.release()
class GCStreamHandler(logging.Handler): def __init__(self): logging.Handler.__init__(self) self._lock = GCLock(threading.RLock()) def get_stream(self): raise AbstractError def emit(self, record): self._lock.acquire() try: stream = self.get_stream() stream.write(self.format(record) + '\n') stream.flush() finally: self._lock.release()
class StatusMonitor(object): def __init__(self, num_jobs): self._result = {} self._lock = GCLock() self._num_jobs = num_jobs def is_finished(self): num_success = sum(imap(lambda jds: self._result.get(jds, 0), [ JobDownloadStatus.JOB_OK, JobDownloadStatus.JOB_ALREADY, JobDownloadStatus.JOB_INCOMPLETE])) return num_success == self._num_jobs def register_file_result(self, jobnum, fi_idx, msg, status, proc=None): if proc: log.log_process(proc, msg=log_intro(jobnum, fi_idx) + msg) else: log.info(log_intro(jobnum, fi_idx) + msg) self._register_result(status) return status # returned file status is actually used later def register_job_result(self, jobnum, msg, status): log.info(log_intro(jobnum) + msg) self._register_result(status) def show_results(self): def _iter_download_results(cls): marker = False for stat in sorted(self._result, key=self._result.get): if self._result[stat] and (stat in cls.enum_value_list): yield {0: cls.enum2str(stat), 1: self._result[stat]} marker = True if marker: yield '=' yield {0: 'Total', 1: self._num_jobs} if self._result: ConsoleTable.create([(0, 'Status'), (1, '')], _iter_download_results(JobDownloadStatus), title='Job status overview') ConsoleTable.create([(0, 'Status'), (1, '')], _iter_download_results(FileDownloadStatus), title='File status overview') def _register_result(self, status): self._lock.acquire() try: self._result[status] = self._result.get(status, 0) + 1 finally: self._lock.release()
def download_monitored(jobNum, output, fileIdx, checkPath, sourcePath, targetPath): copyAbortLock = GCLock() monitorLock = GCLock() monitorLock.acquire() monitor = start_thread('Download monitor %s' % jobNum, transfer_monitor, output, fileIdx, checkPath, monitorLock, copyAbortLock) result = -1 procCP = se_copy(sourcePath, targetPath, tmp = checkPath) while True: if not copyAbortLock.acquire(False): monitor.join() break copyAbortLock.release() result = procCP.status(timeout = 0) if result is not None: monitorLock.release() monitor.join() break time.sleep(0.02) if result != 0: output.error('Unable to copy file from SE!') log.critical('%s\n%s\n', procCP.stdout.read(timeout = 0), procCP.stderr.read(timeout = 0)) return False return True
class ANSIGUI(GUI): def __init__(self, config, workflow): config.set('report', 'BasicReport BarReport') (self._stored_stdout, self._stored_stderr) = (sys.stdout, sys.stderr) GUI.__init__(self, config, workflow) self._reportHeight = 0 self._statusHeight = 1 self._old_message = None self._lock = GCLock(threading.RLock()) # drawing lock self._last_report = 0 self._old_size = None def _draw(self, fun): new_size = self._console.getmaxyx() if self._old_size != new_size: self._old_size = new_size self._schedule_update_layout() self._lock.acquire() self._console.hideCursor() self._console.savePos() try: fun() finally: self._console.loadPos() self._console.showCursor() self._lock.release() # Event handling for resizing def _update_layout(self): (sizey, sizex) = self._console.getmaxyx() self._old_size = (sizey, sizex) self._reportHeight = self._report.getHeight() self._console.erase() self._console.setscrreg( min(self._reportHeight + self._statusHeight + 1, sizey), sizey) utils.printTabular.wraplen = sizex - 5 self._update_all() def _schedule_update_layout(self, sig=None, frame=None): start_thread( 'update layout', self._draw, self._update_layout) # using new thread to ensure RLock is free def _wait(self, timeout): oldHandler = signal.signal(signal.SIGWINCH, self._schedule_update_layout) result = utils.wait(timeout) signal.signal(signal.SIGWINCH, oldHandler) return result def _update_report(self): if time.time() - self._last_report < 1: return self._last_report = time.time() self._console.move(0, 0) self._new_stdout.logged = False self._report.display() self._new_stdout.logged = True def _update_status(self): activity_message = None for activity in Activity.root.get_children(): activity_message = activity.getMessage() + '...' if len(activity_message) > 75: activity_message = activity_message[: 37] + '...' + activity_message[ -35:] self._console.move(self._reportHeight + 1, 0) self._new_stdout.logged = False if self._old_message: self._stored_stdout.write(self._old_message.center(65) + '\r') self._stored_stdout.flush() self._old_message = activity_message if activity_message: self._stored_stdout.write('%s' % activity_message.center(65)) self._stored_stdout.flush() self._new_stdout.logged = True def _update_log(self): self._console.move(self._reportHeight + 2, 0) self._console.eraseDown() self._new_stdout.dump() def _update_all(self): self._last_report = 0 self._update_report() self._update_status() self._update_log() def _schedule_update_report_status(self): self._draw(self._update_report) self._draw(self._update_status) def displayWorkflow(self): if not sys.stdout.isatty(): return self._workflow.process(self._wait) self._console = Console(sys.stdout) self._new_stdout = GUIStream(sys.stdout, self._console, self._lock) self._new_stderr = GUIStream(sys.stderr, self._console, self._lock) Activity.callbacks.append(self._schedule_update_report_status) try: # Main cycle - GUI mode (sys.stdout, sys.stderr) = (self._new_stdout, self._new_stderr) self._console.erase() self._schedule_update_layout() self._workflow.process(self._wait) finally: (sys.stdout, sys.stderr) = (self._stored_stdout, self._stored_stderr) self._console.setscrreg() self._console.erase() self._update_all()
class ANSIGUI(GUI): def __init__(self, config, workflow): config.set('report', 'BasicReport BarReport') (self._stored_stdout, self._stored_stderr) = (sys.stdout, sys.stderr) GUI.__init__(self, config, workflow) self._reportHeight = 0 self._statusHeight = 1 self._old_message = None self._lock = GCLock(threading.RLock()) # drawing lock self._last_report = 0 self._old_size = None def _draw(self, fun): new_size = self._console.getmaxyx() if self._old_size != new_size: self._old_size = new_size self._schedule_update_layout() self._lock.acquire() self._console.hideCursor() self._console.savePos() try: fun() finally: self._console.loadPos() self._console.showCursor() self._lock.release() # Event handling for resizing def _update_layout(self): (sizey, sizex) = self._console.getmaxyx() self._old_size = (sizey, sizex) self._reportHeight = self._report.getHeight() self._console.erase() self._console.setscrreg(min(self._reportHeight + self._statusHeight + 1, sizey), sizey) utils.printTabular.wraplen = sizex - 5 self._update_all() def _schedule_update_layout(self, sig = None, frame = None): start_thread('update layout', self._draw, self._update_layout) # using new thread to ensure RLock is free def _wait(self, timeout): oldHandler = signal.signal(signal.SIGWINCH, self._schedule_update_layout) result = utils.wait(timeout) signal.signal(signal.SIGWINCH, oldHandler) return result def _update_report(self): if time.time() - self._last_report < 1: return self._last_report = time.time() self._console.move(0, 0) self._new_stdout.logged = False self._report.display() self._new_stdout.logged = True def _update_status(self): activity_message = None for activity in Activity.root.get_children(): activity_message = activity.getMessage() + '...' if len(activity_message) > 75: activity_message = activity_message[:37] + '...' + activity_message[-35:] self._console.move(self._reportHeight + 1, 0) self._new_stdout.logged = False if self._old_message: self._stored_stdout.write(self._old_message.center(65) + '\r') self._stored_stdout.flush() self._old_message = activity_message if activity_message: self._stored_stdout.write('%s' % activity_message.center(65)) self._stored_stdout.flush() self._new_stdout.logged = True def _update_log(self): self._console.move(self._reportHeight + 2, 0) self._console.eraseDown() self._new_stdout.dump() def _update_all(self): self._last_report = 0 self._update_report() self._update_status() self._update_log() def _schedule_update_report_status(self): self._draw(self._update_report) self._draw(self._update_status) def displayWorkflow(self): if not sys.stdout.isatty(): return self._workflow.process(self._wait) self._console = Console(sys.stdout) self._new_stdout = GUIStream(sys.stdout, self._console, self._lock) self._new_stderr = GUIStream(sys.stderr, self._console, self._lock) Activity.callbacks.append(self._schedule_update_report_status) try: # Main cycle - GUI mode (sys.stdout, sys.stderr) = (self._new_stdout, self._new_stderr) self._console.erase() self._schedule_update_layout() self._workflow.process(self._wait) finally: (sys.stdout, sys.stderr) = (self._stored_stdout, self._stored_stderr) self._console.setscrreg() self._console.erase() self._update_all()