def __init__(self, infinite=False): if infinite: self.progress = Progress( "[progress.description]{task.description}", TimeElapsedColumn(), TextColumn("{task.completed} items scraped"), SpinnerColumn()) else: self.progress = Progress( "[progress.description]{task.description}", BarColumn(), "[progress.percentage]{task.percentage:>3.0f}%", TimeRemainingColumn(), "/", TimeElapsedColumn()) self.item_type_to_task = {}
def do_speed_test( server: Optional[str] = None, disable: Optional[str] = None, up_threads: Optional[int] = None, dl_threads: Optional[int] = None, ) -> SpeedResult: """ 进行 SpeedTest 测试 服务器列表 ID 可以从这儿获取: https://williamyaps.github.io/wlmjavascript/servercli.html 每次仅允许测试一个服务器 :param server: 期望的服务器ID (ID 来自于 SpeedTest 官网) :param disable: up|down 禁止测试 上传/下载 :param up_threads: 上传线程数量 :param dl_threads: 下载线程数量 """ st = Speedtest() st.get_servers(servers=None if server is None else [server]) server = st.best if disable != "up": with Progress( "[progress.description]{task.description}", BarColumn(), "{task.completed} / {task.total}", TimeElapsedColumn(), ) as progress: task_id = progress.add_task(f"上传: {server['host']}") def up_cb(idx, total, **kwargs): speed_test_cb(progress, task_id, idx, total, **kwargs) st.upload(threads=up_threads, callback=up_cb) if disable != "dl": with Progress( "[progress.description]{task.description}", BarColumn(), "{task.completed} / {task.total}", TimeElapsedColumn(), ) as progress: task_id = progress.add_task(f"下载: {server['host']}") def dl_cb(idx, total, **kwargs): speed_test_cb(progress, task_id, idx, total, **kwargs) st.download(threads=dl_threads, callback=dl_cb) return SpeedResult(**st.results.dict())
class ProgressBar: OPTIONS = [ "[progress.description]{task.description}", BarColumn(), "[progress.percentage]{task.completed:>6}/{task.total}", TimeElapsedColumn(), ] def __init__(self, description, total): self.description = description self.total = total def __enter__(self): self.progress = Progress(*self.OPTIONS) self.progress.start() self.task = self.progress.add_task(self.description, total=self.total) return self def __exit__(self, exc_type, exc_val, exc_tb): self.progress.stop() def print(self, message): self.progress.console.print(message) def advance(self, advance=1): self.progress.update(self.task, advance=advance)
def create_progress_bar(): """ Create progress bar to display overall progress. Returns rich.progress.Progress instance if the Rich Python package is available, or a shim DummyProgress instance otherwise. """ if use_rich() and build_option('show_progress_bar'): # pick random spinner, from a selected subset of available spinner (see 'python3 -m rich.spinner') spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line', 'monkey', 'point', 'simpleDots')) progress_bar = Progress( SpinnerColumn(spinner), "[progress.percentage]{task.percentage:>3.1f}%", TextColumn( "[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total} done)" ), BarColumn(bar_width=None), TimeElapsedColumn(), transient=True, expand=True, ) else: progress_bar = DummyProgress() return progress_bar
def download(self, batch, articles, folder): """ Download a pdf batch """ log_file = os.path.join(folder, 'missing.log') logger.remove() logger.add(log_file, format="{time} {level} {message}", mode='w', level="INFO") assert len(articles) > 0, 'no article.' progress = Progress(TextColumn( "[progress.description]{task.description}", table_column=Column(ratio=1)), TimeElapsedColumn(table_column=Column(ratio=1)), BarColumn(table_column=Column(ratio=2)), "| {task.completed} of {task.total:>2.0f}", expand=False) missing = 0 with progress: task = progress.add_task(f" {batch} |", total=len(articles)) for article in articles: done = self.get_article( article['article_url'], os.path.join(folder, article['file_name'])) if done: progress.update(task, advance=1) else: missing += 1 logger.info("NOT_FOUND_IN_SCI-HUB | " + article['warning_str']) return missing, log_file
def configure_columns(self, trainer) -> list: return [ TextColumn("[progress.description]{task.description}"), BarColumn(), BatchesProcessedColumn(), TimeRemainingColumn(), TimeElapsedColumn() ]
def train_one_epoch(model, optim, loss_fn, loader, epoch, steps, device, writer, global_i, writer_interval=200, normalize=None): model.train() status_col = TextColumn("") running_loss = 0 lr = optim.param_groups[0]['lr'] if normalize is not None: assert len(normalize) == 2, "mean and std values should be provided to use data normalization" fetcher = DataPrefetcher(loader, mean=normalize[0], std=normalize[1], device=device) # modified behavior - w/ input normalization else: fetcher = DataPrefetcher(loader, mean=None, std=None, device=device) # original behavior - no input normalization samples, targets = fetcher.next() with Progress("[progress.description]{task.description}", "[{task.completed}/{task.total}]", BarColumn(), "[progress.percentage]{task.percentage:>3.0f}%", TimeRemainingColumn(), TextColumn("/"), TimeElapsedColumn(), status_col, expand=False, console=CONSOLE, refresh_per_second=5) as progress: task = progress.add_task(description=f'[Epoch {epoch}]', total=steps) i = 0 # counter t_start = time.time() while samples is not None: # zero the parameter gradients optim.zero_grad() # forward + backward + optimize out = model(samples) loss = loss_fn(out, targets) loss.backward() optim.step() # collect running loss running_loss += loss.item() i += 1 global_i += 1 # update tensorboard if i % writer_interval == 0: writer.add_scalar('Loss/Train', running_loss/i, global_i) # pre-fetch next samples samples, targets = fetcher.next() # update trackbar if not progress.finished: status_col.text_format = f"Loss: {running_loss/i:.06f} " \ f"speed: {(time.time() - t_start)/i:.4f}s/it " \ f"lr: {lr}" progress.update(task, advance=1) return running_loss / i, global_i
def extensions_progress_bar(): """ Get progress bar to show progress for installing extensions. """ progress_bar = Progress( TextColumn("[bold blue]{task.description}"), BarColumn(), TimeElapsedColumn(), ) return progress_bar
def __init__(self, *args, **kwargs): columns = [ TextColumn("[progress.description]{task.description}"), BarColumn(), TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), DownloadColumn(), TimeRemainingColumn(), TimeElapsedColumn() ] kwargs['refresh_per_second'] = 1 super().__init__(*columns, *args, **kwargs)
def status_bar(): """ Get progress bar to display overall progress. """ progress_bar = Progress( TimeElapsedColumn(Column(min_width=7, no_wrap=True)), TextColumn( "{task.completed} out of {task.total} easyconfigs done{task.description}" ), ) return progress_bar
def __init__(self, host): """ Create progressbar instance """ self.host = host desc = TextColumn("{task.description}", table_column=Column(ratio=1)) progress = BarColumn(bar_width=None, table_column=Column(ratio=1)) time_elapsed = TimeElapsedColumn(table_column=Column()) self.progress = Progress( desc, progress, "[white][progress.percentage]{task.percentage:>3.0f}%", time_elapsed, expand=True, )
def __init__( self, description: str = 'Working...', total_length: Optional[float] = None, message_on_done: Optional[Union[str, Callable[..., str]]] = None, columns: Optional[Union[str, ProgressColumn]] = None, disable: bool = False, console: Optional[Console] = None, **kwargs, ): """Init a custom progress bar based on rich. This is the default progress bar of jina if you want to customize it you should probably just use a rich `Progress` and add your custom column and task :param description: description of your task ex : 'Working...' :param total_length: the number of steps :param message_on_done: The message that you want to print at the end of your task. It can either be a string to be formatted with task (ex '{task.completed}') task or a function which take task as input (ex : lambda task : f'{task.completed}' :param columns: If you want to customize the column of the progress bar. Note that you should probably directly use rich Progress object than overwriting these columns parameters. :param total_length: disable the progress bar .. # noqa: DAR202 .. # noqa: DAR101 .. # noqa: DAR003 """ def _default_message_on_done(task): return f'{task.completed} steps done in {get_readable_time(seconds=task.finished_time)}' columns = columns or [ SpinnerColumn(), _OnDoneColumn(f'DONE', description, 'progress.description'), BarColumn(complete_style='green', finished_style='yellow'), TimeElapsedColumn(), '[progress.percentage]{task.percentage:>3.0f}%', TextColumn('ETA:', style='progress.remaining'), TimeRemainingColumn(), _OnDoneColumn(message_on_done if message_on_done else _default_message_on_done), ] if not console: console = get_rich_console() super().__init__(*columns, console=console, disable=disable, **kwargs) self.task_id = self.add_task( 'Working...', total=total_length if total_length else 100.0)
def download_all_progress_bar(): """ Get progress bar to show progress on downloading of all source files. """ progress_bar = Progress( TextColumn( "[bold blue]Fetching files: {task.percentage:>3.0f}% ({task.completed}/{task.total})" ), BarColumn(), TimeElapsedColumn(), TextColumn("({task.description})"), ) return progress_bar
def RichTQDM(): return Progress( SpinnerColumn(), "[progress.description]{task.description}", BarColumn(), TextColumn("{task.completed}/{task.total}"), "[", TimeElapsedColumn(), "<", TimeRemainingColumn(), ',', SpeedColumn(), ']', refresh_per_second=.1, speed_estimate_period=30 )
def easyconfig_progress_bar(): """ Get progress bar to display progress for installing a single easyconfig file. """ progress_bar = Progress( SpinnerColumn('point', speed=0.2), TextColumn( "[bold green]{task.description} ({task.completed} out of {task.total} steps done)" ), BarColumn(), TimeElapsedColumn(), refresh_per_second=1, ) return progress_bar
def parse_apk(apkfile, workers, fn_match=None, outfile=None): console = Console() console.log(f"Parsing {apkfile} with {workers} workers ...") dexes = list(extract_dex_files(apkfile)) console.log(f"Found {len(dexes)} DEX file.") total = sum(map(lambda d: len(d.data), dexes)) progress = Progress( TextColumn("[progress.description]{task.description}"), BarColumn( complete_style='bar.complete', finished_style='bar.finished', pulse_style='bar.pulse', ), TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), TimeRemainingColumn(), TimeElapsedColumn(), console=console, ) out = {} out.update(JNI_COMMON) num_classes = 0 t0 = datetime.now() with progress: task = progress.add_task("Analyzing...", total=total) with multiprocessing.Pool(workers) as pool: result = pool.imap(parse_dex_proc, dexes) for dex, count, res in result: if count == 0: console.log( "Parse {} ({} bytes) [bold red]failed: {}".format( dex.name, len(dex.data), res)) continue console.log("Parse {} ({} bytes), found {} classes.".format( dex.name, len(dex.data), count)) num_classes += count progress.update(task, advance=len(dex.data)) for cname, data in res.items(): if fn_match and not fn_match(cname): continue out.update(data) console.log("Aanlyzed {} classes, cost: {}".format(num_classes, datetime.now() - t0)) console.log("Found {} JNI methods.".format(len(out))) if not outfile: console.print_json(data=out) else: with open(outfile, 'w') as f: json.dump(out, f, indent=2, ensure_ascii=False)
def run_councils_with_progress(self): to_run = self.councils_to_run with Progress( "[progress.description]{task.description}", BarColumn(), "[progress.percentage]{task.percentage:>3.0f}%", TimeElapsedColumn(), console=self.console, auto_refresh=False, ) as progress: total = progress.add_task(description=f"Total", total=len(to_run)) while not progress.finished: for council in to_run: self.run_council(council.council_id) progress.update(total, advance=1) progress.refresh()
def test_columns() -> None: console = Console( file=io.StringIO(), force_terminal=True, width=80, log_time_format="[TIME]", color_system="truecolor", legacy_windows=False, log_path=False, _environ={}, ) progress = Progress( "test", TextColumn("{task.description}"), BarColumn(bar_width=None), TimeRemainingColumn(), TimeElapsedColumn(), FileSizeColumn(), TotalFileSizeColumn(), DownloadColumn(), TransferSpeedColumn(), MofNCompleteColumn(), MofNCompleteColumn(separator=" of "), transient=True, console=console, auto_refresh=False, get_time=MockClock(), ) task1 = progress.add_task("foo", total=10) task2 = progress.add_task("bar", total=7) with progress: for n in range(4): progress.advance(task1, 3) progress.advance(task2, 4) print("foo") console.log("hello") console.print("world") progress.refresh() from .render import replace_link_ids result = replace_link_ids(console.file.getvalue()) print(repr(result)) expected = "\x1b[?25ltest foo \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:07\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \x1b[32m 0/10\x1b[0m \x1b[32m 0 of 10\x1b[0m\ntest bar \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:18\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \x1b[32m0/7 \x1b[0m \x1b[32m0 of 7 \x1b[0m\r\x1b[2K\x1b[1A\x1b[2Kfoo\ntest foo \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:07\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \x1b[32m 0/10\x1b[0m \x1b[32m 0 of 10\x1b[0m\ntest bar \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:18\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \x1b[32m0/7 \x1b[0m \x1b[32m0 of 7 \x1b[0m\r\x1b[2K\x1b[1A\x1b[2K\x1b[2;36m[TIME]\x1b[0m\x1b[2;36m \x1b[0mhello \ntest foo \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:07\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \x1b[32m 0/10\x1b[0m \x1b[32m 0 of 10\x1b[0m\ntest bar \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:18\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \x1b[32m0/7 \x1b[0m \x1b[32m0 of 7 \x1b[0m\r\x1b[2K\x1b[1A\x1b[2Kworld\ntest foo \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:07\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \x1b[32m 0/10\x1b[0m \x1b[32m 0 of 10\x1b[0m\ntest bar \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:18\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \x1b[32m0/7 \x1b[0m \x1b[32m0 of 7 \x1b[0m\r\x1b[2K\x1b[1A\x1b[2Ktest foo \x1b[38;2;114;156;31m━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[33m0:00:34\x1b[0m \x1b[32m12 \x1b[0m \x1b[32m10 \x1b[0m \x1b[32m12/10 \x1b[0m \x1b[31m1 \x1b[0m \x1b[32m12/10\x1b[0m \x1b[32m12 of 10\x1b[0m\n \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[31mbyte/s \x1b[0m \ntest bar \x1b[38;2;114;156;31m━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[33m0:00:29\x1b[0m \x1b[32m16 \x1b[0m \x1b[32m7 bytes\x1b[0m \x1b[32m16/7 \x1b[0m \x1b[31m2 \x1b[0m \x1b[32m16/7 \x1b[0m \x1b[32m16 of 7 \x1b[0m\n \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[31mbytes/s\x1b[0m \r\x1b[2K\x1b[1A\x1b[2K\x1b[1A\x1b[2K\x1b[1A\x1b[2Ktest foo \x1b[38;2;114;156;31m━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[33m0:00:34\x1b[0m \x1b[32m12 \x1b[0m \x1b[32m10 \x1b[0m \x1b[32m12/10 \x1b[0m \x1b[31m1 \x1b[0m \x1b[32m12/10\x1b[0m \x1b[32m12 of 10\x1b[0m\n \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[31mbyte/s \x1b[0m \ntest bar \x1b[38;2;114;156;31m━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[33m0:00:29\x1b[0m \x1b[32m16 \x1b[0m \x1b[32m7 bytes\x1b[0m \x1b[32m16/7 \x1b[0m \x1b[31m2 \x1b[0m \x1b[32m16/7 \x1b[0m \x1b[32m16 of 7 \x1b[0m\n \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[31mbytes/s\x1b[0m \n\x1b[?25h\r\x1b[1A\x1b[2K\x1b[1A\x1b[2K\x1b[1A\x1b[2K\x1b[1A\x1b[2K" assert result == expected
def start_progress_bar(self, of=None): """ Starts the progress bar for the current epoch. Args: of: The total number of batches in the epoch or None if that is unknown. """ if of is None: if self.n_batches_last is None: of = 0 else: of = self.n_batches_last if self.progress is None: self.progress = Progress( TextColumn("Epoch {task.fields[epoch]}"), SpinnerColumn(), TextColumn( "Batch {task.fields[batch]:3} / {task.fields[of]:3}"), BarColumn(), TimeElapsedColumn(table_column=Column(header="Elapsed")), TextColumn("/"), TimeRemainingColumn(table_column=Column(header="Remaining")), TextColumn("|"), TextColumn("[red]Loss: {task.fields[batch_loss]:1.3f}[red],"), TextColumn( "[red][b]Avg.: {task.fields[running_mean]:1.3f}[/b][/red]" ), auto_refresh=False, transient=True, ) fields = { "epoch": 1, "running_mean": np.nan, "batch_loss": np.nan, "of": of, "batch": 0, } self.task = self.progress.add_task(f"Epoch {self.i_epoch + 1}", total=of, **fields) self.progress.update(self.task, completed=0, total=of, epoch=self.i_epoch + 1, mode="training")
def evaluate(model, loss_fn, loader, epoch, steps, device, normalize=None): model.eval() status_col = TextColumn("") running_loss = 0 if normalize is not None: assert len(normalize) == 2, "mean and std values should be provided to use data normalization" fetcher = DataPrefetcher(loader, mean=normalize[0], std=normalize[1], device=device) # modified behavior - w/ input normalization else: fetcher = DataPrefetcher(loader, mean=None, std=None, device=device) # original behavior - no input normalization samples, targets = fetcher.next() with Progress("[progress.description]{task.description}", "[{task.completed}/{task.total}]", BarColumn(), "[progress.percentage]{task.percentage:>3.0f}%", TimeRemainingColumn(), TextColumn("/"), TimeElapsedColumn(), status_col, expand=False, console=CONSOLE, refresh_per_second=5) as progress: task = progress.add_task(description=f'[Eval {epoch}]', total=steps) i = 0 # counter t_start = time.time() with torch.no_grad(): while samples is not None: # forward only out = model(samples) val_loss = loss_fn(out, targets) # collect running loss running_loss += val_loss.item() i += 1 # pre-fetch next samples samples, targets = fetcher.next() if not progress.finished: status_col.text_format = f"Val loss: {running_loss/i:.06f} " \ f"speed: {(time.time() - t_start)/i:.4f}s/it" progress.update(task, advance=1) return running_loss / i
def rich_bar(): return Progress( TextColumn("[bold blue]{task.description}", justify="right"), "[progress.percentage]{task.percentage:>3.0f}%", BarColumn(bar_width=None), TextColumn( "[bold blue][progress.completed]{task.completed}/{task.total}", justify="right", ), "[", TimeElapsedColumn(), "<", TimeRemainingColumn(), "|", SpeedColumn(".1f"), TextColumn("[progress.data.speed]{task.fields[unit]}/s", justify="right"), "]", auto_refresh=False, )
def __init__(self, *args, **kwargs): """ This class accepts the following parameters *in addition* to the parameters accepted by `tqdm`. Parameters ---------- progress : tuple, optional arguments for `rich.progress.Progress()`. options : dict, optional keyword arguments for `rich.progress.Progress()`. """ kwargs = kwargs.copy() kwargs['gui'] = True # convert disable = None to False kwargs['disable'] = bool(kwargs.get('disable', False)) progress = kwargs.pop('progress', None) options = kwargs.pop('options', {}).copy() super(tqdm_rich, self).__init__(*args, **kwargs) if self.disable: return warn("rich is experimental/alpha", TqdmExperimentalWarning, stacklevel=2) d = self.format_dict if progress is None: progress = ("[progress.description]{task.description}" "[progress.percentage]{task.percentage:>4.0f}%", BarColumn(bar_width=None), FractionColumn(unit_scale=d['unit_scale'], unit_divisor=d['unit_divisor']), "[", TimeElapsedColumn(), "<", TimeRemainingColumn(), ",", RateColumn(unit=d['unit'], unit_scale=d['unit_scale'], unit_divisor=d['unit_divisor']), "]") options.setdefault('transient', not self.leave) self._prog = Progress(*progress, **options) self._prog.__enter__() self._task_id = self._prog.add_task(self.desc or "", **d)
def __init__(self, num_tests: int, progress_styles: List[TestProgressStyle]): super().__init__(num_tests, progress_styles) self.spinner_column = SpinnerColumn( style="pass.textonly", finished_text=GREEN_CHECK, ) self.bar_column = BarColumn( complete_style="pass.textonly", finished_style="pass.textonly", ) self.progress: Optional[Progress] = Progress( self.spinner_column, TimeElapsedColumn(), self.bar_column, "[progress.percentage]{task.percentage:>3.0f}%", "[progress.percentage][{task.completed} / {task.total}]", console=self.console, ) self.task = self.progress.add_task("Testing...", total=num_tests)
def MTT_statistics(args): from src.data.dataset import MTTDataset from rich.progress import Progress, BarColumn, TimeRemainingColumn, TimeElapsedColumn, TextColumn # initialize statistics dataset = MTTDataset(path=args.p_data, split=args.split) total_segments = len(dataset) n_samples_per_segment = args.n_samples sum_x, sum_x2 = 0, 0 n_samples_total = total_segments * n_samples_per_segment # iterating over dataset with Progress("[progress.description]{task.description}", BarColumn(), "[progress.percentage]{task.percentage:>3.0f}%", TimeRemainingColumn(), TextColumn("/"), TimeElapsedColumn(), "{task.completed} of {task.total} steps", expand=False, console=CONSOLE, refresh_per_second=5) as progress: task = progress.add_task(description='[Scanning Dataset] ', total=total_segments) for i in range(total_segments): sample, label = dataset[i] sum_x += np.sum(sample) sum_x2 += np.sum(sample**2) progress.update(task, advance=1) LOG.info("Calculating final results...") mean = sum_x / n_samples_total std = np.sqrt( (sum_x2 - sum_x**2 / n_samples_total) / (n_samples_total - 1)) LOG.info(f"Mean: {mean}\nStddev: {std}") return
def test_using_default_columns() -> None: # can only check types, as the instances do not '==' each other expected_default_types = [ TextColumn, BarColumn, TaskProgressColumn, TimeRemainingColumn, ] progress = Progress() assert [type(c) for c in progress.columns] == expected_default_types progress = Progress( SpinnerColumn(), *Progress.get_default_columns(), "Elapsed:", TimeElapsedColumn(), ) assert [type(c) for c in progress.columns] == [ SpinnerColumn, *expected_default_types, str, TimeElapsedColumn, ]
class RichTransferProgress(RichProgress): SUMMARY_COLS = ( TextColumn("[magenta]{task.description}[bold green]"), MofNCompleteColumnWithUnit(), TimeElapsedColumn(), ) TRANSFER_COLS = ( TextColumn(" [blue]{task.description}"), BarColumn(), DownloadColumn(), TransferSpeedColumn(), TextColumn("eta"), TimeRemainingColumn(), ) def get_renderables(self): summary_tasks, other_tasks = split( lambda task: task.fields.get("progress_type") == "summary", self.tasks, ) self.columns = self.SUMMARY_COLS yield self.make_tasks_table(summary_tasks) self.columns = self.TRANSFER_COLS yield self.make_tasks_table(other_tasks)
yield self.make_tasks_table([task]) if task.fields.get("progress_type") == "status": self.columns = progress_bar_status yield self.make_tasks_table([task]) progress_bar_overall = ( "[bold green]{task.description}:", SpinnerColumn(), BarColumn(bar_width=None, complete_style="green"), "[progress.percentage]{task.percentage:>3.0f}%", "• Files: [progress.percentage]{task.completed} / {task.total}", # "• Remaining:", # TimeRemainingColumn(), "• Time Elapsed:", TimeElapsedColumn(), ) progress_bar_shortname = (TextColumn(" " * 4 + "[blue]{task.fields[name]}"), ) progress_bar_status = ( TextColumn(" " * 8 + "{task.fields[status]}:"), BarColumn(bar_width=20, complete_style="green"), "[progress.percentage]{task.percentage:>3.0f}%", "• Time Elapsed:", TimeElapsedColumn(), "• {task.fields[name]} [progress.percentage]{task.completed:>4} / {task.total:>4}", ) #%%
def event(jid="all", tag=None, progress="log", stop_signal=None): """ Function to listen to events emitted by Nornir Proxy Minions. Matched event printed to terminal. :param tag: (str) tag regex string, default is ``nornir\-proxy/.*`` :param jid: (int, str) Job ID to listen events for, default is ``all`` :param progress: (str) progress display mode - ``log``, ``raw``, ``bars``, ``tree`` :param stop_signal: (obj) thread Event object, stops listening to events if ``stop_signal.is_set()``, if ``stop_signal is None``, listens and print events until keyboard interrupt hit - ``ctrl+c`` ``bars`` and ``tree`` progress display modes use Rich library, to properly display various symbols and characters need to make sure to use utf-8 encoding for your environment for example by running these commands:: [root@salt-master ~]# PYTHONIOENCODING=utf-8 [root@salt-master ~]# export PYTHONIOENCODING """ # start rich console globals().setdefault("console", Console()) stop_signal = stop_signal or Event() events_queue = queue.Queue() tag = ( tag if tag else ( r"nornir\-proxy\/.*" if jid == "all" else r"nornir\-proxy\/{jid}\/.*".format(jid=jid) ) ) # start events thread listen_events_thread = Thread( target=_get_salt_nornir_event, kwargs={"stop_signal": stop_signal, "tag": tag, "events_queue": events_queue}, daemon=True, # to not block once main process finishes ) listen_events_thread.start() # display events if HAS_RICH and progress == "bars": tasks = {} stop_events_loop = Event() rich_progress = Progress( TextColumn("{task.description}"), BarColumn(), TextColumn("{task.percentage:>3.0f}%"), TextColumn("{task.completed}/{task.total}"), TimeElapsedColumn(), TextColumn("{task.fields[info]}"), TextColumn("{task.fields[status]}"), refresh_per_second=5, ) # listen to events indefinitely if stop_signal is None with rich_progress: while True: try: e = events_queue.get(timeout=1) except queue.Empty: continue finally: # check if need to finish if stop_signal.is_set() and all( [t.finished for t in rich_progress.tasks] ): stop_events_loop.set() break edata = e["data"] jid = edata["jid"] task_type = edata["task_type"] task_event = edata["task_event"] status = edata["status"] task_name = edata["task_name"] status_str = ( f"[red]{status}" if status == "FAILED" else f"[green]{status}" ) description_str = ( "[blue]{timestamp}[/]:{user}:{jid}:[bright_magenta]{function}[/]" ) # catch task started events and add new progress bar if task_type == "task" and task_event == "started": # if job runs on multiple proxy minions if jid in tasks: task = rich_progress.tasks[tasks[jid]] task.total += len(edata["hosts"]) rich_progress.update(tasks[jid]) else: total = len(edata["hosts"]) description = description_str.format(**edata) info = "[cyan]{}[/]:{}".format( task_type, task_name.split(".")[-1] ) tasks[jid] = rich_progress.add_task( description, total=total, info=info, status=status ) # catch task instance end events to advance progress bar elif task_type == "task_instance" and task_event == "completed": if jid in tasks: task = rich_progress.tasks[tasks[jid]] if status == "PASSED": rich_progress.update(tasks[jid], advance=1) if task.completed >= task.total or status == "FAILED": rich_progress.tasks[tasks[jid]].fields[ "status" ] = status_str elif task_type == "task" and task_event == "completed": if jid in tasks: task = rich_progress.tasks[tasks[jid]] if task.completed >= task.total or status == "FAILED": rich_progress.tasks[tasks[jid]].fields[ "status" ] = status_str rich_progress.stop_task(tasks[jid]) # stop all subtasks for this jid and update their status for task in tasks: if task.startswith(jid): rich_progress.tasks[tasks[task]].fields[ "status" ] = status_str rich_progress.stop_task(tasks[task]) # handle subtask progress elif task_type == "subtask" and task_event == "started": tid = "{jid}:{task_name}".format(**edata) if tid not in tasks and jid in tasks: total = rich_progress.tasks[tasks[jid]].total description = description_str.format(**edata) info = "[cyan]{task_type}[/]:{task_name}".format(**edata) tasks[tid] = rich_progress.add_task( description, total=total, info=info, status=status ) # update task total if got additional start events elif tid in tasks and jid in tasks: task = rich_progress.tasks[tasks[tid]] task.total = rich_progress.tasks[tasks[jid]].total rich_progress.update(tasks[tid]) elif task_type == "subtask" and task_event == "completed": tid = "{jid}:{task_name}".format(**edata) if tid in tasks: task = rich_progress.tasks[tasks[tid]] if status == "PASSED": rich_progress.update(tasks[tid], advance=1) if task.completed >= task.total or status == "FAILED": rich_progress.tasks[tasks[tid]].fields[ "status" ] = status_str # hide tasks beyond vertical overflow point hght = console.size.height - 2 for tindx, _ in enumerate(rich_progress.tasks[:-hght]): rich_progress.update(tindx, visible=False) elif progress == "tree": pass elif progress == "raw": while not stop_signal.is_set(): try: e = events_queue.get(timeout=1) print(e) except queue.Empty: continue # handle 'log' progress mode else: while not stop_signal.is_set(): try: e = events_queue.get(timeout=1) except queue.Empty: continue edata = e["data"] # form message string and print it if edata["task_type"] == "task": edata["hs"] = ", ".join(edata["hosts"]) elif edata["task_type"] in ["task_instance", "subtask"]: edata["hs"] = edata["host"] msg = "{timestamp}:{user}:{jid}:{proxy_id}:w{worker_id} {function} {hs} {task_type} {task_event} {task_name}; {status}" print(msg.format(**edata))
def run_tests( tests: List[str], sequential: bool = typer.Option( False, '--sequential', '-s', help='Run all test of each group in order'), workers: int = typer.Option(1, '--workers', '-p', help='Number of parallel tasks'), iterations: int = typer.Option(10, '--iter', '-n', help='Number of iterations to run'), output: Optional[Path] = typer.Option(None, '--output', '-o', help='Output path to use'), verbose: bool = typer.Option(False, '--verbose', '-v', help='Verbosity level'), archive: bool = typer.Option( False, '--archive', '-a', help='Save all logs intead of only failed ones'), race: bool = typer.Option(False, '--race/--no-race', '-r/-R', help='Run with race checker'), loop: bool = typer.Option(False, '--loop', '-l', help='Run continuously'), growth: int = typer.Option( 10, '--growth', '-g', help='Growth ratio of iterations when using --loop'), timing: bool = typer.Option(False, '--timing', '-t', help='Report timing, only works on macOS'), # fmt: on ): print(tests) if output is None: timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") output = Path(timestamp) if race: print("[yellow]Running with the race detector\n[/yellow]") if verbose: print(f"[yellow]Verbosity level set to 1 [/yellow]") os.environ['DEBUG'] = '1' while True: total = iterations * len(tests) completed = 0 results = {test: defaultdict(StatsMeter) for test in tests} if sequential: test_instances = itertools.chain.from_iterable( itertools.repeat(test, iterations) for test in tests) else: test_instances = itertools.chain.from_iterable( itertools.repeat(tests, iterations)) test_instances = iter(test_instances) total_progress = Progress( "[progress.description]{task.description}", BarColumn(), TimeRemainingColumn(), "[progress.percentage]{task.percentage:>3.0f}%", TimeElapsedColumn(), ) total_task = total_progress.add_task("[yellow]Tests[/yellow]", total=total) task_progress = Progress( "[progress.description]{task.description}", SpinnerColumn(), BarColumn(), "{task.completed}/{task.total}", ) tasks = { test: task_progress.add_task(test, total=iterations) for test in tests } progress_table = Table.grid() progress_table.add_row(total_progress) progress_table.add_row(Panel.fit(task_progress)) with Live(progress_table, transient=True) as live: def handler(_, frame): live.stop() print('\n') print_results(results) sys.exit(1) signal.signal(signal.SIGINT, handler) with ThreadPoolExecutor(max_workers=workers) as executor: futures = [] while completed < total: n = len(futures) if n < workers: for test in itertools.islice(test_instances, workers - n): futures.append( executor.submit(run_test, test, race, timing)) done, not_done = wait(futures, return_when=FIRST_COMPLETED) for future in done: test, path, rc, runtime = future.result() results[test]['completed'].add(1) results[test]['time'].add(runtime) task_progress.update(tasks[test], advance=1) dest = (output / f"{test}_{completed}.log").as_posix() if rc != 0: print(f"Failed test {test} - {dest}") task_progress.update( tasks[test], description=f"[red]{test}[/red]") results[test]['failed'].add(1) else: if results[test][ 'completed'].n == iterations and results[ test]['failed'].n == 0: task_progress.update( tasks[test], description=f"[green]{test}[/green]") if rc != 0 or archive: output.mkdir(exist_ok=True, parents=True) shutil.copy(path, dest) if timing: line = last_line(path) real, _, user, _, system, _ = line.replace( ' ' * 8, '').split(' ') results[test]['real_time'].add(float(real)) results[test]['user_time'].add(float(user)) results[test]['system_time'].add(float(system)) os.remove(path) completed += 1 total_progress.update(total_task, advance=1) futures = list(not_done) print_results(results, timing) if loop: iterations *= growth print(f"[yellow]Increasing iterations to {iterations}[/yellow]") else: break
def test_time_elapsed_column(): column = TimeElapsedColumn() task = Task(1, "test", 100, 20, _get_time=lambda: 1.0) text = column.render(task) assert str(text) == "-:--:--"