def __init__(self, infinite=False):
if infinite:
self.progress = Progress(
"[progress.description]{task.description}",
TimeElapsedColumn(),
TextColumn("{task.completed} items scraped"), SpinnerColumn())
else:
self.progress = Progress(
"[progress.description]{task.description}", BarColumn(),
"[progress.percentage]{task.percentage:>3.0f}%",
TimeRemainingColumn(), "/", TimeElapsedColumn())
self.item_type_to_task = {}
def do_speed_test(
server: Optional[str] = None,
disable: Optional[str] = None,
up_threads: Optional[int] = None,
dl_threads: Optional[int] = None,
) -> SpeedResult:
"""
进行 SpeedTest 测试
服务器列表 ID 可以从这儿获取: https://williamyaps.github.io/wlmjavascript/servercli.html
每次仅允许测试一个服务器
:param server: 期望的服务器ID (ID 来自于 SpeedTest 官网)
:param disable: up|down 禁止测试 上传/下载
:param up_threads: 上传线程数量
:param dl_threads: 下载线程数量
"""
st = Speedtest()
st.get_servers(servers=None if server is None else [server])
server = st.best
if disable != "up":
with Progress(
"[progress.description]{task.description}",
BarColumn(),
"{task.completed} / {task.total}",
TimeElapsedColumn(),
) as progress:
task_id = progress.add_task(f"上传: {server['host']}")
def up_cb(idx, total, **kwargs):
speed_test_cb(progress, task_id, idx, total, **kwargs)
st.upload(threads=up_threads, callback=up_cb)
if disable != "dl":
with Progress(
"[progress.description]{task.description}",
BarColumn(),
"{task.completed} / {task.total}",
TimeElapsedColumn(),
) as progress:
task_id = progress.add_task(f"下载: {server['host']}")
def dl_cb(idx, total, **kwargs):
speed_test_cb(progress, task_id, idx, total, **kwargs)
st.download(threads=dl_threads, callback=dl_cb)
return SpeedResult(**st.results.dict())
class ProgressBar:
OPTIONS = [
"[progress.description]{task.description}",
BarColumn(),
"[progress.percentage]{task.completed:>6}/{task.total}",
TimeElapsedColumn(),
]
def __init__(self, description, total):
self.description = description
self.total = total
def __enter__(self):
self.progress = Progress(*self.OPTIONS)
self.progress.start()
self.task = self.progress.add_task(self.description, total=self.total)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.progress.stop()
def print(self, message):
self.progress.console.print(message)
def advance(self, advance=1):
self.progress.update(self.task, advance=advance)
def create_progress_bar():
"""
Create progress bar to display overall progress.
Returns rich.progress.Progress instance if the Rich Python package is available,
or a shim DummyProgress instance otherwise.
"""
if use_rich() and build_option('show_progress_bar'):
# pick random spinner, from a selected subset of available spinner (see 'python3 -m rich.spinner')
spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line',
'monkey', 'point', 'simpleDots'))
progress_bar = Progress(
SpinnerColumn(spinner),
"[progress.percentage]{task.percentage:>3.1f}%",
TextColumn(
"[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total} done)"
),
BarColumn(bar_width=None),
TimeElapsedColumn(),
transient=True,
expand=True,
)
else:
progress_bar = DummyProgress()
return progress_bar
def download(self, batch, articles, folder):
"""
Download a pdf batch
"""
log_file = os.path.join(folder, 'missing.log')
logger.remove()
logger.add(log_file,
format="{time} {level} {message}",
mode='w',
level="INFO")
assert len(articles) > 0, 'no article.'
progress = Progress(TextColumn(
"[progress.description]{task.description}",
table_column=Column(ratio=1)),
TimeElapsedColumn(table_column=Column(ratio=1)),
BarColumn(table_column=Column(ratio=2)),
"| {task.completed} of {task.total:>2.0f}",
expand=False)
missing = 0
with progress:
task = progress.add_task(f" {batch} |", total=len(articles))
for article in articles:
done = self.get_article(
article['article_url'],
os.path.join(folder, article['file_name']))
if done:
progress.update(task, advance=1)
else:
missing += 1
logger.info("NOT_FOUND_IN_SCI-HUB | " +
article['warning_str'])
return missing, log_file
def configure_columns(self, trainer) -> list:
return [
TextColumn("[progress.description]{task.description}"),
BarColumn(),
BatchesProcessedColumn(),
TimeRemainingColumn(),
TimeElapsedColumn()
]
def train_one_epoch(model, optim, loss_fn, loader, epoch, steps, device,
writer, global_i, writer_interval=200, normalize=None):
model.train()
status_col = TextColumn("")
running_loss = 0
lr = optim.param_groups[0]['lr']
if normalize is not None:
assert len(normalize) == 2, "mean and std values should be provided to use data normalization"
fetcher = DataPrefetcher(loader, mean=normalize[0], std=normalize[1], device=device) # modified behavior - w/ input normalization
else:
fetcher = DataPrefetcher(loader, mean=None, std=None, device=device) # original behavior - no input normalization
samples, targets = fetcher.next()
with Progress("[progress.description]{task.description}",
"[{task.completed}/{task.total}]",
BarColumn(),
"[progress.percentage]{task.percentage:>3.0f}%",
TimeRemainingColumn(),
TextColumn("/"),
TimeElapsedColumn(),
status_col,
expand=False, console=CONSOLE, refresh_per_second=5) as progress:
task = progress.add_task(description=f'[Epoch {epoch}]', total=steps)
i = 0 # counter
t_start = time.time()
while samples is not None:
# zero the parameter gradients
optim.zero_grad()
# forward + backward + optimize
out = model(samples)
loss = loss_fn(out, targets)
loss.backward()
optim.step()
# collect running loss
running_loss += loss.item()
i += 1
global_i += 1
# update tensorboard
if i % writer_interval == 0:
writer.add_scalar('Loss/Train', running_loss/i, global_i)
# pre-fetch next samples
samples, targets = fetcher.next()
# update trackbar
if not progress.finished:
status_col.text_format = f"Loss: {running_loss/i:.06f} " \
f"speed: {(time.time() - t_start)/i:.4f}s/it " \
f"lr: {lr}"
progress.update(task, advance=1)
return running_loss / i, global_i
def extensions_progress_bar():
"""
Get progress bar to show progress for installing extensions.
"""
progress_bar = Progress(
TextColumn("[bold blue]{task.description}"),
BarColumn(),
TimeElapsedColumn(),
)
return progress_bar
def __init__(self, *args, **kwargs):
columns = [
TextColumn("[progress.description]{task.description}"),
BarColumn(),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
DownloadColumn(),
TimeRemainingColumn(),
TimeElapsedColumn()
]
kwargs['refresh_per_second'] = 1
super().__init__(*columns, *args, **kwargs)
def status_bar():
"""
Get progress bar to display overall progress.
"""
progress_bar = Progress(
TimeElapsedColumn(Column(min_width=7, no_wrap=True)),
TextColumn(
"{task.completed} out of {task.total} easyconfigs done{task.description}"
),
)
return progress_bar
def __init__(self, host):
""" Create progressbar instance """
self.host = host
desc = TextColumn("{task.description}", table_column=Column(ratio=1))
progress = BarColumn(bar_width=None, table_column=Column(ratio=1))
time_elapsed = TimeElapsedColumn(table_column=Column())
self.progress = Progress(
desc,
progress,
"[white][progress.percentage]{task.percentage:>3.0f}%",
time_elapsed,
expand=True,
)
def __init__(
self,
description: str = 'Working...',
total_length: Optional[float] = None,
message_on_done: Optional[Union[str, Callable[..., str]]] = None,
columns: Optional[Union[str, ProgressColumn]] = None,
disable: bool = False,
console: Optional[Console] = None,
**kwargs,
):
"""Init a custom progress bar based on rich. This is the default progress bar of jina if you want to customize
it you should probably just use a rich `Progress` and add your custom column and task
:param description: description of your task ex : 'Working...'
:param total_length: the number of steps
:param message_on_done: The message that you want to print at the end of your task. It can either be a string to
be formatted with task (ex '{task.completed}') task or a function which take task as input (ex : lambda task : f'{task.completed}'
:param columns: If you want to customize the column of the progress bar. Note that you should probably directly use
rich Progress object than overwriting these columns parameters.
:param total_length: disable the progress bar
.. # noqa: DAR202
.. # noqa: DAR101
.. # noqa: DAR003
"""
def _default_message_on_done(task):
return f'{task.completed} steps done in {get_readable_time(seconds=task.finished_time)}'
columns = columns or [
SpinnerColumn(),
_OnDoneColumn(f'DONE', description, 'progress.description'),
BarColumn(complete_style='green', finished_style='yellow'),
TimeElapsedColumn(),
'[progress.percentage]{task.percentage:>3.0f}%',
TextColumn('ETA:', style='progress.remaining'),
TimeRemainingColumn(),
_OnDoneColumn(message_on_done
if message_on_done else _default_message_on_done),
]
if not console:
console = get_rich_console()
super().__init__(*columns, console=console, disable=disable, **kwargs)
self.task_id = self.add_task(
'Working...', total=total_length if total_length else 100.0)
def download_all_progress_bar():
"""
Get progress bar to show progress on downloading of all source files.
"""
progress_bar = Progress(
TextColumn(
"[bold blue]Fetching files: {task.percentage:>3.0f}% ({task.completed}/{task.total})"
),
BarColumn(),
TimeElapsedColumn(),
TextColumn("({task.description})"),
)
return progress_bar
def RichTQDM():
return Progress(
SpinnerColumn(),
"[progress.description]{task.description}",
BarColumn(),
TextColumn("{task.completed}/{task.total}"),
"[",
TimeElapsedColumn(),
"<",
TimeRemainingColumn(),
',',
SpeedColumn(),
']',
refresh_per_second=.1, speed_estimate_period=30
)
def easyconfig_progress_bar():
"""
Get progress bar to display progress for installing a single easyconfig file.
"""
progress_bar = Progress(
SpinnerColumn('point', speed=0.2),
TextColumn(
"[bold green]{task.description} ({task.completed} out of {task.total} steps done)"
),
BarColumn(),
TimeElapsedColumn(),
refresh_per_second=1,
)
return progress_bar
def parse_apk(apkfile, workers, fn_match=None, outfile=None):
console = Console()
console.log(f"Parsing {apkfile} with {workers} workers ...")
dexes = list(extract_dex_files(apkfile))
console.log(f"Found {len(dexes)} DEX file.")
total = sum(map(lambda d: len(d.data), dexes))
progress = Progress(
TextColumn("[progress.description]{task.description}"),
BarColumn(
complete_style='bar.complete',
finished_style='bar.finished',
pulse_style='bar.pulse',
),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
TimeRemainingColumn(),
TimeElapsedColumn(),
console=console,
)
out = {}
out.update(JNI_COMMON)
num_classes = 0
t0 = datetime.now()
with progress:
task = progress.add_task("Analyzing...", total=total)
with multiprocessing.Pool(workers) as pool:
result = pool.imap(parse_dex_proc, dexes)
for dex, count, res in result:
if count == 0:
console.log(
"Parse {} ({} bytes) [bold red]failed: {}".format(
dex.name, len(dex.data), res))
continue
console.log("Parse {} ({} bytes), found {} classes.".format(
dex.name, len(dex.data), count))
num_classes += count
progress.update(task, advance=len(dex.data))
for cname, data in res.items():
if fn_match and not fn_match(cname):
continue
out.update(data)
console.log("Aanlyzed {} classes, cost: {}".format(num_classes,
datetime.now() - t0))
console.log("Found {} JNI methods.".format(len(out)))
if not outfile:
console.print_json(data=out)
else:
with open(outfile, 'w') as f:
json.dump(out, f, indent=2, ensure_ascii=False)
def run_councils_with_progress(self):
to_run = self.councils_to_run
with Progress(
"[progress.description]{task.description}",
BarColumn(),
"[progress.percentage]{task.percentage:>3.0f}%",
TimeElapsedColumn(),
console=self.console,
auto_refresh=False,
) as progress:
total = progress.add_task(description=f"Total", total=len(to_run))
while not progress.finished:
for council in to_run:
self.run_council(council.council_id)
progress.update(total, advance=1)
progress.refresh()
def test_columns() -> None:
console = Console(
file=io.StringIO(),
force_terminal=True,
width=80,
log_time_format="[TIME]",
color_system="truecolor",
legacy_windows=False,
log_path=False,
_environ={},
)
progress = Progress(
"test",
TextColumn("{task.description}"),
BarColumn(bar_width=None),
TimeRemainingColumn(),
TimeElapsedColumn(),
FileSizeColumn(),
TotalFileSizeColumn(),
DownloadColumn(),
TransferSpeedColumn(),
MofNCompleteColumn(),
MofNCompleteColumn(separator=" of "),
transient=True,
console=console,
auto_refresh=False,
get_time=MockClock(),
)
task1 = progress.add_task("foo", total=10)
task2 = progress.add_task("bar", total=7)
with progress:
for n in range(4):
progress.advance(task1, 3)
progress.advance(task2, 4)
print("foo")
console.log("hello")
console.print("world")
progress.refresh()
from .render import replace_link_ids
result = replace_link_ids(console.file.getvalue())
print(repr(result))
expected = "\x1b[?25ltest foo \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:07\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \x1b[32m 0/10\x1b[0m \x1b[32m 0 of 10\x1b[0m\ntest bar \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:18\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \x1b[32m0/7 \x1b[0m \x1b[32m0 of 7 \x1b[0m\r\x1b[2K\x1b[1A\x1b[2Kfoo\ntest foo \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:07\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \x1b[32m 0/10\x1b[0m \x1b[32m 0 of 10\x1b[0m\ntest bar \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:18\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \x1b[32m0/7 \x1b[0m \x1b[32m0 of 7 \x1b[0m\r\x1b[2K\x1b[1A\x1b[2K\x1b[2;36m[TIME]\x1b[0m\x1b[2;36m \x1b[0mhello \ntest foo \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:07\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \x1b[32m 0/10\x1b[0m \x1b[32m 0 of 10\x1b[0m\ntest bar \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:18\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \x1b[32m0/7 \x1b[0m \x1b[32m0 of 7 \x1b[0m\r\x1b[2K\x1b[1A\x1b[2Kworld\ntest foo \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:07\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \x1b[32m 0/10\x1b[0m \x1b[32m 0 of 10\x1b[0m\ntest bar \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:18\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \x1b[32m0/7 \x1b[0m \x1b[32m0 of 7 \x1b[0m\r\x1b[2K\x1b[1A\x1b[2Ktest foo \x1b[38;2;114;156;31m━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[33m0:00:34\x1b[0m \x1b[32m12 \x1b[0m \x1b[32m10 \x1b[0m \x1b[32m12/10 \x1b[0m \x1b[31m1 \x1b[0m \x1b[32m12/10\x1b[0m \x1b[32m12 of 10\x1b[0m\n \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[31mbyte/s \x1b[0m \ntest bar \x1b[38;2;114;156;31m━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[33m0:00:29\x1b[0m \x1b[32m16 \x1b[0m \x1b[32m7 bytes\x1b[0m \x1b[32m16/7 \x1b[0m \x1b[31m2 \x1b[0m \x1b[32m16/7 \x1b[0m \x1b[32m16 of 7 \x1b[0m\n \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[31mbytes/s\x1b[0m \r\x1b[2K\x1b[1A\x1b[2K\x1b[1A\x1b[2K\x1b[1A\x1b[2Ktest foo \x1b[38;2;114;156;31m━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[33m0:00:34\x1b[0m \x1b[32m12 \x1b[0m \x1b[32m10 \x1b[0m \x1b[32m12/10 \x1b[0m \x1b[31m1 \x1b[0m \x1b[32m12/10\x1b[0m \x1b[32m12 of 10\x1b[0m\n \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[31mbyte/s \x1b[0m \ntest bar \x1b[38;2;114;156;31m━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[33m0:00:29\x1b[0m \x1b[32m16 \x1b[0m \x1b[32m7 bytes\x1b[0m \x1b[32m16/7 \x1b[0m \x1b[31m2 \x1b[0m \x1b[32m16/7 \x1b[0m \x1b[32m16 of 7 \x1b[0m\n \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[31mbytes/s\x1b[0m \n\x1b[?25h\r\x1b[1A\x1b[2K\x1b[1A\x1b[2K\x1b[1A\x1b[2K\x1b[1A\x1b[2K"
assert result == expected
def start_progress_bar(self, of=None):
"""
Starts the progress bar for the current epoch.
Args:
of: The total number of batches in the epoch or None
if that is unknown.
"""
if of is None:
if self.n_batches_last is None:
of = 0
else:
of = self.n_batches_last
if self.progress is None:
self.progress = Progress(
TextColumn("Epoch {task.fields[epoch]}"),
SpinnerColumn(),
TextColumn(
"Batch {task.fields[batch]:3} / {task.fields[of]:3}"),
BarColumn(),
TimeElapsedColumn(table_column=Column(header="Elapsed")),
TextColumn("/"),
TimeRemainingColumn(table_column=Column(header="Remaining")),
TextColumn("|"),
TextColumn("[red]Loss: {task.fields[batch_loss]:1.3f}[red],"),
TextColumn(
"[red][b]Avg.: {task.fields[running_mean]:1.3f}[/b][/red]"
),
auto_refresh=False,
transient=True,
)
fields = {
"epoch": 1,
"running_mean": np.nan,
"batch_loss": np.nan,
"of": of,
"batch": 0,
}
self.task = self.progress.add_task(f"Epoch {self.i_epoch + 1}",
total=of,
**fields)
self.progress.update(self.task,
completed=0,
total=of,
epoch=self.i_epoch + 1,
mode="training")
def evaluate(model, loss_fn, loader, epoch, steps, device, normalize=None):
model.eval()
status_col = TextColumn("")
running_loss = 0
if normalize is not None:
assert len(normalize) == 2, "mean and std values should be provided to use data normalization"
fetcher = DataPrefetcher(loader, mean=normalize[0], std=normalize[1], device=device) # modified behavior - w/ input normalization
else:
fetcher = DataPrefetcher(loader, mean=None, std=None, device=device) # original behavior - no input normalization
samples, targets = fetcher.next()
with Progress("[progress.description]{task.description}",
"[{task.completed}/{task.total}]",
BarColumn(),
"[progress.percentage]{task.percentage:>3.0f}%",
TimeRemainingColumn(),
TextColumn("/"),
TimeElapsedColumn(),
status_col,
expand=False, console=CONSOLE, refresh_per_second=5) as progress:
task = progress.add_task(description=f'[Eval {epoch}]', total=steps)
i = 0 # counter
t_start = time.time()
with torch.no_grad():
while samples is not None:
# forward only
out = model(samples)
val_loss = loss_fn(out, targets)
# collect running loss
running_loss += val_loss.item()
i += 1
# pre-fetch next samples
samples, targets = fetcher.next()
if not progress.finished:
status_col.text_format = f"Val loss: {running_loss/i:.06f} " \
f"speed: {(time.time() - t_start)/i:.4f}s/it"
progress.update(task, advance=1)
return running_loss / i
def rich_bar():
return Progress(
TextColumn("[bold blue]{task.description}", justify="right"),
"[progress.percentage]{task.percentage:>3.0f}%",
BarColumn(bar_width=None),
TextColumn(
"[bold blue][progress.completed]{task.completed}/{task.total}",
justify="right",
),
"[",
TimeElapsedColumn(),
"<",
TimeRemainingColumn(),
"|",
SpeedColumn(".1f"),
TextColumn("[progress.data.speed]{task.fields[unit]}/s",
justify="right"),
"]",
auto_refresh=False,
)
def __init__(self, *args, **kwargs):
"""
This class accepts the following parameters *in addition* to
the parameters accepted by `tqdm`.
Parameters
----------
progress : tuple, optional
arguments for `rich.progress.Progress()`.
options : dict, optional
keyword arguments for `rich.progress.Progress()`.
"""
kwargs = kwargs.copy()
kwargs['gui'] = True
# convert disable = None to False
kwargs['disable'] = bool(kwargs.get('disable', False))
progress = kwargs.pop('progress', None)
options = kwargs.pop('options', {}).copy()
super(tqdm_rich, self).__init__(*args, **kwargs)
if self.disable:
return
warn("rich is experimental/alpha",
TqdmExperimentalWarning,
stacklevel=2)
d = self.format_dict
if progress is None:
progress = ("[progress.description]{task.description}"
"[progress.percentage]{task.percentage:>4.0f}%",
BarColumn(bar_width=None),
FractionColumn(unit_scale=d['unit_scale'],
unit_divisor=d['unit_divisor']), "[",
TimeElapsedColumn(), "<", TimeRemainingColumn(), ",",
RateColumn(unit=d['unit'],
unit_scale=d['unit_scale'],
unit_divisor=d['unit_divisor']), "]")
options.setdefault('transient', not self.leave)
self._prog = Progress(*progress, **options)
self._prog.__enter__()
self._task_id = self._prog.add_task(self.desc or "", **d)
def __init__(self, num_tests: int, progress_styles: List[TestProgressStyle]):
super().__init__(num_tests, progress_styles)
self.spinner_column = SpinnerColumn(
style="pass.textonly",
finished_text=GREEN_CHECK,
)
self.bar_column = BarColumn(
complete_style="pass.textonly",
finished_style="pass.textonly",
)
self.progress: Optional[Progress] = Progress(
self.spinner_column,
TimeElapsedColumn(),
self.bar_column,
"[progress.percentage]{task.percentage:>3.0f}%",
"[progress.percentage][{task.completed} / {task.total}]",
console=self.console,
)
self.task = self.progress.add_task("Testing...", total=num_tests)
def MTT_statistics(args):
from src.data.dataset import MTTDataset
from rich.progress import Progress, BarColumn, TimeRemainingColumn, TimeElapsedColumn, TextColumn
# initialize statistics
dataset = MTTDataset(path=args.p_data, split=args.split)
total_segments = len(dataset)
n_samples_per_segment = args.n_samples
sum_x, sum_x2 = 0, 0
n_samples_total = total_segments * n_samples_per_segment
# iterating over dataset
with Progress("[progress.description]{task.description}",
BarColumn(),
"[progress.percentage]{task.percentage:>3.0f}%",
TimeRemainingColumn(),
TextColumn("/"),
TimeElapsedColumn(),
"{task.completed} of {task.total} steps",
expand=False,
console=CONSOLE,
refresh_per_second=5) as progress:
task = progress.add_task(description='[Scanning Dataset] ',
total=total_segments)
for i in range(total_segments):
sample, label = dataset[i]
sum_x += np.sum(sample)
sum_x2 += np.sum(sample**2)
progress.update(task, advance=1)
LOG.info("Calculating final results...")
mean = sum_x / n_samples_total
std = np.sqrt(
(sum_x2 - sum_x**2 / n_samples_total) / (n_samples_total - 1))
LOG.info(f"Mean: {mean}\nStddev: {std}")
return
def test_using_default_columns() -> None:
# can only check types, as the instances do not '==' each other
expected_default_types = [
TextColumn,
BarColumn,
TaskProgressColumn,
TimeRemainingColumn,
]
progress = Progress()
assert [type(c) for c in progress.columns] == expected_default_types
progress = Progress(
SpinnerColumn(),
*Progress.get_default_columns(),
"Elapsed:",
TimeElapsedColumn(),
)
assert [type(c) for c in progress.columns] == [
SpinnerColumn,
*expected_default_types,
str,
TimeElapsedColumn,
]
class RichTransferProgress(RichProgress):
SUMMARY_COLS = (
TextColumn("[magenta]{task.description}[bold green]"),
MofNCompleteColumnWithUnit(),
TimeElapsedColumn(),
)
TRANSFER_COLS = (
TextColumn(" [blue]{task.description}"),
BarColumn(),
DownloadColumn(),
TransferSpeedColumn(),
TextColumn("eta"),
TimeRemainingColumn(),
)
def get_renderables(self):
summary_tasks, other_tasks = split(
lambda task: task.fields.get("progress_type") == "summary",
self.tasks,
)
self.columns = self.SUMMARY_COLS
yield self.make_tasks_table(summary_tasks)
self.columns = self.TRANSFER_COLS
yield self.make_tasks_table(other_tasks)
yield self.make_tasks_table([task])
if task.fields.get("progress_type") == "status":
self.columns = progress_bar_status
yield self.make_tasks_table([task])
progress_bar_overall = (
"[bold green]{task.description}:",
SpinnerColumn(),
BarColumn(bar_width=None, complete_style="green"),
"[progress.percentage]{task.percentage:>3.0f}%",
"• Files: [progress.percentage]{task.completed} / {task.total}",
# "• Remaining:",
# TimeRemainingColumn(),
"• Time Elapsed:",
TimeElapsedColumn(),
)
progress_bar_shortname = (TextColumn(" " * 4 + "[blue]{task.fields[name]}"), )
progress_bar_status = (
TextColumn(" " * 8 + "{task.fields[status]}:"),
BarColumn(bar_width=20, complete_style="green"),
"[progress.percentage]{task.percentage:>3.0f}%",
"• Time Elapsed:",
TimeElapsedColumn(),
"• {task.fields[name]} [progress.percentage]{task.completed:>4} / {task.total:>4}",
)
#%%
def event(jid="all", tag=None, progress="log", stop_signal=None):
"""
Function to listen to events emitted by Nornir Proxy Minions. Matched
event printed to terminal.
:param tag: (str) tag regex string, default is ``nornir\-proxy/.*``
:param jid: (int, str) Job ID to listen events for, default is ``all``
:param progress: (str) progress display mode - ``log``, ``raw``, ``bars``, ``tree``
:param stop_signal: (obj) thread Event object, stops listening to events if ``stop_signal.is_set()``,
if ``stop_signal is None``, listens and print events until keyboard interrupt hit - ``ctrl+c``
``bars`` and ``tree`` progress display modes use Rich library, to properly display various
symbols and characters need to make sure to use utf-8 encoding for your environment for example
by running these commands::
[root@salt-master ~]# PYTHONIOENCODING=utf-8
[root@salt-master ~]# export PYTHONIOENCODING
"""
# start rich console
globals().setdefault("console", Console())
stop_signal = stop_signal or Event()
events_queue = queue.Queue()
tag = (
tag
if tag
else (
r"nornir\-proxy\/.*"
if jid == "all"
else r"nornir\-proxy\/{jid}\/.*".format(jid=jid)
)
)
# start events thread
listen_events_thread = Thread(
target=_get_salt_nornir_event,
kwargs={"stop_signal": stop_signal, "tag": tag, "events_queue": events_queue},
daemon=True, # to not block once main process finishes
)
listen_events_thread.start()
# display events
if HAS_RICH and progress == "bars":
tasks = {}
stop_events_loop = Event()
rich_progress = Progress(
TextColumn("{task.description}"),
BarColumn(),
TextColumn("{task.percentage:>3.0f}%"),
TextColumn("{task.completed}/{task.total}"),
TimeElapsedColumn(),
TextColumn("{task.fields[info]}"),
TextColumn("{task.fields[status]}"),
refresh_per_second=5,
)
# listen to events indefinitely if stop_signal is None
with rich_progress:
while True:
try:
e = events_queue.get(timeout=1)
except queue.Empty:
continue
finally:
# check if need to finish
if stop_signal.is_set() and all(
[t.finished for t in rich_progress.tasks]
):
stop_events_loop.set()
break
edata = e["data"]
jid = edata["jid"]
task_type = edata["task_type"]
task_event = edata["task_event"]
status = edata["status"]
task_name = edata["task_name"]
status_str = (
f"[red]{status}" if status == "FAILED" else f"[green]{status}"
)
description_str = (
"[blue]{timestamp}[/]:{user}:{jid}:[bright_magenta]{function}[/]"
)
# catch task started events and add new progress bar
if task_type == "task" and task_event == "started":
# if job runs on multiple proxy minions
if jid in tasks:
task = rich_progress.tasks[tasks[jid]]
task.total += len(edata["hosts"])
rich_progress.update(tasks[jid])
else:
total = len(edata["hosts"])
description = description_str.format(**edata)
info = "[cyan]{}[/]:{}".format(
task_type, task_name.split(".")[-1]
)
tasks[jid] = rich_progress.add_task(
description, total=total, info=info, status=status
)
# catch task instance end events to advance progress bar
elif task_type == "task_instance" and task_event == "completed":
if jid in tasks:
task = rich_progress.tasks[tasks[jid]]
if status == "PASSED":
rich_progress.update(tasks[jid], advance=1)
if task.completed >= task.total or status == "FAILED":
rich_progress.tasks[tasks[jid]].fields[
"status"
] = status_str
elif task_type == "task" and task_event == "completed":
if jid in tasks:
task = rich_progress.tasks[tasks[jid]]
if task.completed >= task.total or status == "FAILED":
rich_progress.tasks[tasks[jid]].fields[
"status"
] = status_str
rich_progress.stop_task(tasks[jid])
# stop all subtasks for this jid and update their status
for task in tasks:
if task.startswith(jid):
rich_progress.tasks[tasks[task]].fields[
"status"
] = status_str
rich_progress.stop_task(tasks[task])
# handle subtask progress
elif task_type == "subtask" and task_event == "started":
tid = "{jid}:{task_name}".format(**edata)
if tid not in tasks and jid in tasks:
total = rich_progress.tasks[tasks[jid]].total
description = description_str.format(**edata)
info = "[cyan]{task_type}[/]:{task_name}".format(**edata)
tasks[tid] = rich_progress.add_task(
description, total=total, info=info, status=status
)
# update task total if got additional start events
elif tid in tasks and jid in tasks:
task = rich_progress.tasks[tasks[tid]]
task.total = rich_progress.tasks[tasks[jid]].total
rich_progress.update(tasks[tid])
elif task_type == "subtask" and task_event == "completed":
tid = "{jid}:{task_name}".format(**edata)
if tid in tasks:
task = rich_progress.tasks[tasks[tid]]
if status == "PASSED":
rich_progress.update(tasks[tid], advance=1)
if task.completed >= task.total or status == "FAILED":
rich_progress.tasks[tasks[tid]].fields[
"status"
] = status_str
# hide tasks beyond vertical overflow point
hght = console.size.height - 2
for tindx, _ in enumerate(rich_progress.tasks[:-hght]):
rich_progress.update(tindx, visible=False)
elif progress == "tree":
pass
elif progress == "raw":
while not stop_signal.is_set():
try:
e = events_queue.get(timeout=1)
print(e)
except queue.Empty:
continue
# handle 'log' progress mode
else:
while not stop_signal.is_set():
try:
e = events_queue.get(timeout=1)
except queue.Empty:
continue
edata = e["data"]
# form message string and print it
if edata["task_type"] == "task":
edata["hs"] = ", ".join(edata["hosts"])
elif edata["task_type"] in ["task_instance", "subtask"]:
edata["hs"] = edata["host"]
msg = "{timestamp}:{user}:{jid}:{proxy_id}:w{worker_id} {function} {hs} {task_type} {task_event} {task_name}; {status}"
print(msg.format(**edata))
def run_tests(
tests: List[str],
sequential: bool = typer.Option(
False,
'--sequential',
'-s',
help='Run all test of each group in order'),
workers: int = typer.Option(1,
'--workers',
'-p',
help='Number of parallel tasks'),
iterations: int = typer.Option(10,
'--iter',
'-n',
help='Number of iterations to run'),
output: Optional[Path] = typer.Option(None,
'--output',
'-o',
help='Output path to use'),
verbose: bool = typer.Option(False,
'--verbose',
'-v',
help='Verbosity level'),
archive: bool = typer.Option(
False,
'--archive',
'-a',
help='Save all logs intead of only failed ones'),
race: bool = typer.Option(False,
'--race/--no-race',
'-r/-R',
help='Run with race checker'),
loop: bool = typer.Option(False, '--loop', '-l', help='Run continuously'),
growth: int = typer.Option(
10,
'--growth',
'-g',
help='Growth ratio of iterations when using --loop'),
timing: bool = typer.Option(False,
'--timing',
'-t',
help='Report timing, only works on macOS'),
# fmt: on
):
print(tests)
if output is None:
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
output = Path(timestamp)
if race:
print("[yellow]Running with the race detector\n[/yellow]")
if verbose:
print(f"[yellow]Verbosity level set to 1 [/yellow]")
os.environ['DEBUG'] = '1'
while True:
total = iterations * len(tests)
completed = 0
results = {test: defaultdict(StatsMeter) for test in tests}
if sequential:
test_instances = itertools.chain.from_iterable(
itertools.repeat(test, iterations) for test in tests)
else:
test_instances = itertools.chain.from_iterable(
itertools.repeat(tests, iterations))
test_instances = iter(test_instances)
total_progress = Progress(
"[progress.description]{task.description}",
BarColumn(),
TimeRemainingColumn(),
"[progress.percentage]{task.percentage:>3.0f}%",
TimeElapsedColumn(),
)
total_task = total_progress.add_task("[yellow]Tests[/yellow]",
total=total)
task_progress = Progress(
"[progress.description]{task.description}",
SpinnerColumn(),
BarColumn(),
"{task.completed}/{task.total}",
)
tasks = {
test: task_progress.add_task(test, total=iterations)
for test in tests
}
progress_table = Table.grid()
progress_table.add_row(total_progress)
progress_table.add_row(Panel.fit(task_progress))
with Live(progress_table, transient=True) as live:
def handler(_, frame):
live.stop()
print('\n')
print_results(results)
sys.exit(1)
signal.signal(signal.SIGINT, handler)
with ThreadPoolExecutor(max_workers=workers) as executor:
futures = []
while completed < total:
n = len(futures)
if n < workers:
for test in itertools.islice(test_instances,
workers - n):
futures.append(
executor.submit(run_test, test, race, timing))
done, not_done = wait(futures, return_when=FIRST_COMPLETED)
for future in done:
test, path, rc, runtime = future.result()
results[test]['completed'].add(1)
results[test]['time'].add(runtime)
task_progress.update(tasks[test], advance=1)
dest = (output / f"{test}_{completed}.log").as_posix()
if rc != 0:
print(f"Failed test {test} - {dest}")
task_progress.update(
tasks[test], description=f"[red]{test}[/red]")
results[test]['failed'].add(1)
else:
if results[test][
'completed'].n == iterations and results[
test]['failed'].n == 0:
task_progress.update(
tasks[test],
description=f"[green]{test}[/green]")
if rc != 0 or archive:
output.mkdir(exist_ok=True, parents=True)
shutil.copy(path, dest)
if timing:
line = last_line(path)
real, _, user, _, system, _ = line.replace(
' ' * 8, '').split(' ')
results[test]['real_time'].add(float(real))
results[test]['user_time'].add(float(user))
results[test]['system_time'].add(float(system))
os.remove(path)
completed += 1
total_progress.update(total_task, advance=1)
futures = list(not_done)
print_results(results, timing)
if loop:
iterations *= growth
print(f"[yellow]Increasing iterations to {iterations}[/yellow]")
else:
break
def test_time_elapsed_column():
column = TimeElapsedColumn()
task = Task(1, "test", 100, 20, _get_time=lambda: 1.0)
text = column.render(task)
assert str(text) == "-:--:--"