def test_spinner_column():
column = SpinnerColumn()
column.set_spinner("dots2")
task = Task(1, "test", 100, 20, _get_time=lambda: 1.0)
result = column.render(task)
print(repr(result))
expected = "⡿"
assert str(result) == expected
def get_bolumler(self, isim):
""" Animenin bölümlerini {bölüm,title} formatında döndürür. """
with Progress(SpinnerColumn(),
'[progress.description]{task.description}',
BarColumn(bar_width=40)) as progress:
task = progress.add_task("[cyan]Bölümler getiriliyor..",
start=False)
anime_slug = self.tamliste[isim]
raw = self.driver.execute_script(
f"return $.get('/anime/{anime_slug}')")
soup = bs4(raw, "html.parser")
self.anime_ismi = soup.title.text
anime_code = soup.find('meta', {
'name': 'twitter:image'
}).get('content').split('lerb/')[1][:-4]
raw = self.driver.execute_script(
f"return $.get('https://www.turkanime.net/ajax/bolumler&animeId={anime_code}')"
)
soup = bs4(raw, "html.parser")
bolumler = []
for bolum in soup.findAll("span", {"class": "bolumAdi"}):
bolumler.append({
'name':
bolum.text,
'value':
bolum.findParent().get("href").split("video/")[1]
})
progress.update(task, visible=False)
return bolumler
def put(
self, path: str, key: str, acl: str = "public-read", metadata: dict = {}
) -> dict:
with Progress(
SpinnerColumn(spinner_name="earth"),
TextColumn("[progress.description]{task.description}"),
BarColumn(bar_width=30),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
TimeRemainingColumn(),
transient=True,
) as progress:
size = os.stat(path).st_size
task = progress.add_task(
f"[green]Uploading [bold]{os.path.basename(path)}[/bold]", total=size
)
def update_progress(complete):
progress.update(task, completed=complete)
try:
response = self.client.upload_file(
path,
self.bucket,
key,
ExtraArgs={"ACL": acl, "Metadata": metadata},
Callback=update_progress,
)
except ClientError as e:
logging.error(e)
return {}
return response
def list(storage_prefix):
"""lists all available assets and their versions."""
manager = StorageProvider(prefix=storage_prefix, )
console = Console()
tree = Tree("[bold]Assets store[/bold]")
tree.add(
f"[dim]storage provider[/dim] {manager.driver.__class__.__name__}")
tree.add(f"[dim]prefix[/dim] {storage_prefix}")
console.print(tree)
table = Table(show_header=True, header_style="bold")
table.add_column("Asset name")
table.add_column("Versions", style="dim")
n = 0
n_versions = 0
with Progress(SpinnerColumn(),
"[progress.description]{task.description}",
transient=True) as progress:
progress.add_task("Listing remote assets", start=False)
for asset_name, versions_list in manager.iterate_assets():
table.add_row(asset_name, " ".join(versions_list))
n += 1
n_versions += len(versions_list)
console.print(table)
console.print(f"Found {n} assets ({n_versions} different versions)")
def check_remote_servers(logger, servers):
check_result = dict()
progress = Progress(
"{task.description}",
SpinnerColumn(),
BarColumn(),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
)
with progress:
with ThreadPoolExecutor() as pool:
for server in servers:
if server["check"] == "False":
continue
reader = DriverReader(progress)
tinfo = pool.submit(
reader.get_remote_drivers,
server["ip"],
server["user"],
server["password"],
server["ssh_port"],
server["query"],
)
check_result[server["ip"]] = tinfo
for ip in check_result:
drivers, wu_drivers, noinfo_drivers = check_result[ip].result()
check_result[ip] = {
"drivers": drivers,
"weak-update-drivers": wu_drivers,
"noinfo-drivers": noinfo_drivers,
}
progress.console.print("[green]Check completed![/]")
return check_result
def create_progress_bar():
"""
Create progress bar to display overall progress.
Returns rich.progress.Progress instance if the Rich Python package is available,
or a shim DummyProgress instance otherwise.
"""
if use_rich() and build_option('show_progress_bar'):
# pick random spinner, from a selected subset of available spinner (see 'python3 -m rich.spinner')
spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line',
'monkey', 'point', 'simpleDots'))
progress_bar = Progress(
SpinnerColumn(spinner),
"[progress.percentage]{task.percentage:>3.1f}%",
TextColumn(
"[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total} done)"
),
BarColumn(bar_width=None),
TimeElapsedColumn(),
transient=True,
expand=True,
)
else:
progress_bar = DummyProgress()
return progress_bar
def create_and_remove_task(self, *args, **kwargs):
message = ""
with Progress(
"[bold]{task.description}",
SpinnerColumn(spinner_name="dots12", style="white"),
) as progress:
# Determine spinner text
if func.__name__ == "remove_all":
description = f"Removing all files in project {self.project}..."
elif func.__name__ == "remove_file":
description = "Removing file(s)..."
elif func.__name__ == "remove_folder":
description = "Removing folder(s)..."
# Add progress task
task = progress.add_task(description=description)
# Execute function
message = func(self, *args, **kwargs)
# Remove progress task
progress.remove_task(task)
# Printout removal response
if message is None:
rm_type = "File" if func.__name__ == "remove_file" else "Folder"
message = f"{rm_type}(s) successfully removed."
console = rich.console.Console()
console.print(message)
def progress(self) -> Progress:
"""
Return a pre-configured rich spinner.
"""
text_column = TextColumn("{task.description}")
spinner_column = SpinnerColumn("simpleDotsScrolling",
style="bold white")
return Progress(text_column, spinner_column, transient=True)
def main():
with Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
transient=True,
) as progress:
progress.add_task(description="Processing...", total=None)
progress.add_task(description="Preparing...", total=None)
time.sleep(5)
print("Done!")
def get_default_columns(cls) -> Tuple[ProgressColumn, ...]:
return (
TextColumn("[progress.description]{task.description}"),
MofNCompleteColumn(),
BarColumn(bar_width=30),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
TimeElapsedColumn(),
TimeRemainingColumn(),
RateColumn(),
SpinnerColumn(),
)
def cli(output, summary, password, include_all, gains, gains_112a, force_pdfminer, filename):
"""CLI function."""
output_ext = None
if output is not None:
output_ext = os.path.splitext(output)[-1].lower()
if not (summary or gains or output_ext in (".csv", ".json")):
summary = True
try:
with Progress(
SpinnerColumn(spinner_name="clock"),
TextColumn("[progress.description]{task.description}"),
BarColumn(pulse_style="yellow"),
transient=True,
) as progress:
progress.add_task("Reading CAS file", start=False, total=10.0)
data = read_cas_pdf(filename, password, force_pdfminer=force_pdfminer)
except ParserException as exc:
console.print(f"Error parsing pdf file :: [bold red]{str(exc)}[/]")
sys.exit(1)
if summary:
print_summary(
data,
include_zero_folios=include_all,
output_filename=None if output_ext in (".csv", ".json") else output,
)
if output_ext in (".csv", ".json"):
if output_ext == ".csv":
if summary or data["cas_type"] == CASFileType.SUMMARY.name:
description = "Generating summary CSV file..."
conv_fn = cas2csv_summary
else:
description = "Generating detailed CSV file..."
conv_fn = cas2csv
else:
description = "Generating JSON file..."
conv_fn = cas2json
console.print(description)
with open(output, "w", newline="", encoding="utf-8") as fp:
fp.write(conv_fn(data))
console.print(f"File saved : [bold]{output}[/]")
if gains or gains_112a:
try:
print_gains(
data,
output_file_path=output if output_ext == ".csv" else None,
gains_112a=gains_112a,
)
except IncompleteCASError:
console.print("[bold red]Error![/] - Cannot compute gains. CAS is incomplete!")
sys.exit(2)
except GainsError as exc:
console.print(exc)
def __init__(self, infinite=False):
if infinite:
self.progress = Progress(
"[progress.description]{task.description}",
TimeElapsedColumn(),
TextColumn("{task.completed} items scraped"), SpinnerColumn())
else:
self.progress = Progress(
"[progress.description]{task.description}", BarColumn(),
"[progress.percentage]{task.percentage:>3.0f}%",
TimeRemainingColumn(), "/", TimeElapsedColumn())
self.item_type_to_task = {}
def create_progress():
"""Returns a Rich Progress class."""
return Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}",
table_column=Column(ratio=1)),
BarColumn(bar_width=None, table_column=Column(ratio=1)),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
CustomTimeElapsedColumn(),
expand=True,
)
def track(
sequence: Union[Sequence[ProgressType], Iterable[ProgressType]],
description: str = "",
total: Optional[float] = None,
auto_refresh: bool = True,
console: Optional[Console] = None,
transient: bool = False,
get_time: Optional[Callable[[], float]] = None,
refresh_per_second: float = 10,
style: StyleType = "bar.back",
complete_style: StyleType = "bar.complete",
finished_style: StyleType = "bar.finished",
pulse_style: StyleType = "bar.pulse",
update_period: float = 0.1,
disable: bool = False,
remove: bool = False,
) -> Iterable[ProgressType]:
columns: List[ProgressColumn] = ([
TextColumn("[progress.description]{task.description}")
] if description else []) + [
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
BarColumn(
bar_width=30,
style=style,
complete_style=complete_style,
finished_style=finished_style,
pulse_style=pulse_style,
),
TimeElapsedColumn(),
TimeRemainingColumn(),
RateColumn(),
SpinnerColumn(),
]
progress = Progress(
*columns,
auto_refresh=auto_refresh,
console=console,
transient=transient,
get_time=get_time,
refresh_per_second=refresh_per_second or 10,
disable=disable,
)
with progress:
yield from progress.track(
sequence,
total=total,
description=description,
update_period=update_period,
remove=remove,
)
def __init__(
self,
numRepos: int,
follower: int = 0,
following: int = 0,
numStat: int = 5,
) -> None:
self.numStat = numStat
self.numRepos = numRepos
self._profileText = Text(
f"{follower:03d} Follower\n{following:03d} Following\n{numRepos:03d} Public Repositories"
)
self.progressTable = Table.grid(expand=True)
self.progressTotal = Progress(
"{task.description}",
SpinnerColumn(),
BarColumn(),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
)
self.progressTable.add_row(
Panel(
Align.center(Text(
"""Placeholder""",
justify="center",
)),
title="[b]Info",
border_style="red",
padding=(1, 1),
),
Panel(
Align.center(self._profileText),
title="[b]Profile Info",
border_style="yellow",
padding=(1, 1),
),
Panel(
self.progressTotal, # type:ignore
title="[b]Total Progress",
border_style="green",
padding=(1, 2),
),
)
self.taskTotal = self.progressTotal.add_task(description="Progress",
total=numStat * numRepos)
self.taskRepo = self.progressTotal.add_task(
description="Repository [bold yellow]#", total=numRepos)
self.taskStat = self.progressTotal.add_task(
description="Stat [bold violet]#", total=numStat)
def progress_bar(cls, tasks_range: int) -> Progress:
job_progress = Progress(
"{task.description}",
SpinnerColumn("dots", finished_text="Done", style="cyan"),
BarColumn(),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
)
job_progress.add_task("[Closing]:", total=tasks_range)
progress_table = Table.grid()
progress_table.add_row(
Panel.fit(job_progress, title="[b]jobs", padding=(2, 2)))
return job_progress
def __init__(
self,
description: str = 'Working...',
total_length: Optional[float] = None,
message_on_done: Optional[Union[str, Callable[..., str]]] = None,
columns: Optional[Union[str, ProgressColumn]] = None,
disable: bool = False,
console: Optional[Console] = None,
**kwargs,
):
"""Init a custom progress bar based on rich. This is the default progress bar of jina if you want to customize
it you should probably just use a rich `Progress` and add your custom column and task
:param description: description of your task ex : 'Working...'
:param total_length: the number of steps
:param message_on_done: The message that you want to print at the end of your task. It can either be a string to
be formatted with task (ex '{task.completed}') task or a function which take task as input (ex : lambda task : f'{task.completed}'
:param columns: If you want to customize the column of the progress bar. Note that you should probably directly use
rich Progress object than overwriting these columns parameters.
:param total_length: disable the progress bar
.. # noqa: DAR202
.. # noqa: DAR101
.. # noqa: DAR003
"""
def _default_message_on_done(task):
return f'{task.completed} steps done in {get_readable_time(seconds=task.finished_time)}'
columns = columns or [
SpinnerColumn(),
_OnDoneColumn(f'DONE', description, 'progress.description'),
BarColumn(complete_style='green', finished_style='yellow'),
TimeElapsedColumn(),
'[progress.percentage]{task.percentage:>3.0f}%',
TextColumn('ETA:', style='progress.remaining'),
TimeRemainingColumn(),
_OnDoneColumn(message_on_done
if message_on_done else _default_message_on_done),
]
if not console:
console = get_rich_console()
super().__init__(*columns, console=console, disable=disable, **kwargs)
self.task_id = self.add_task(
'Working...', total=total_length if total_length else 100.0)
def setup(self, trainer, pl_module, stage):
self.progress = Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
BarColumn(),
BatchesProcessedColumn(),
"[",
CustomTimeColumn(),
ProcessingSpeedColumn(),
MetricsTextColumn(trainer, stage),
"]",
console=self.console,
refresh_per_second=self.refresh_rate,
).__enter__()
def easyconfig_progress_bar():
"""
Get progress bar to display progress for installing a single easyconfig file.
"""
progress_bar = Progress(
SpinnerColumn('point', speed=0.2),
TextColumn(
"[bold green]{task.description} ({task.completed} out of {task.total} steps done)"
),
BarColumn(),
TimeElapsedColumn(),
refresh_per_second=1,
)
return progress_bar
def RichTQDM():
return Progress(
SpinnerColumn(),
"[progress.description]{task.description}",
BarColumn(),
TextColumn("{task.completed}/{task.total}"),
"[",
TimeElapsedColumn(),
"<",
TimeRemainingColumn(),
',',
SpeedColumn(),
']',
refresh_per_second=.1, speed_estimate_period=30
)
def __init__(self):
self.__progress = Progress(
SpinnerColumn(),
TextColumn(
"[progress.description]{task.description} {task.fields[current]}"
),
"•",
BarColumn(),
"•",
TransferSpeedColumn(),
TimeRemainingColumn(),
console=Console(),
transient=True,
expand=True)
self.__console = self.__progress.console
def _run(threads, tasks_queue, results, timeout, output_handlers):
results_queue = normal_queue()
finish_tasks_num = 0
tasks_num = tasks_queue.qsize()
with Progress(SpinnerColumn(choice(SPINNER_KEYS)), "{task.completed} / {task.total}", BarColumn(complete_style="cyan"), " ({task.description})", transient=True, console=CONSOLE) as bar:
# create bar_task
bar_task = None
if not CONFIG.option.debug:
bar_task = bar.add_task(
"[cyan]working...", total=tasks_queue.qsize())
# create futures
tasks = [Thread(target=_work, args=(
bar, bar_task, tasks_queue, results_queue, timeout)) for _ in range(threads)]
for task in tasks:
task.daemon = True
task.start()
# get result as completed
try:
while finish_tasks_num < tasks_num:
try:
target, poc, poc_result = results_queue.get(True, timeout)
except Empty:
continue
if target and poc and poc_result:
finish_tasks_num += 1
if not CONFIG.option.debug:
bar.update(bar_task, advance=1)
status = poc_result.get("status", -1)
if status == 0:
logger_func = logger.info
RESULTS.success += 1
elif status == 1:
logger_func = logger.warning
RESULTS.failed += 1
elif status == -1:
logger_func = logger.error
RESULTS.error += 1
logger_func("_run: done poc: %s for %s" %
(poc.name, target))
results[target][poc.name] = poc_result
# handle result
_output(output_handlers, poc, poc_result)
except KeyboardInterrupt:
tasks_queue.queue.clear()
return results
def progress_step(self, step_key, step_name, total):
"""Context manager to show a progress bar during a step execution."""
step_progress = Progress(
'{task.description}',
SpinnerColumn(spinner_name='earth'),
BarColumn(),
TextColumn('[progress.percentage]{task.percentage:>3.0f}%'),
)
step_job = step_progress.add_task(step_name, total=total)
self.progress_bars[step_key] = {
C.PROGRESS_KEY: step_progress,
C.JOB_KEY: step_job,
}
self.progress_table.add_row(step_progress)
yield
step_progress.update(step_job, completed=step_progress.tasks[0].total)
def __init__(self, num_tests: int, progress_styles: List[TestProgressStyle]):
super().__init__(num_tests, progress_styles)
self.spinner_column = SpinnerColumn(
style="pass.textonly",
finished_text=GREEN_CHECK,
)
self.test_description_column = RenderableColumn(Text(""))
self.progress = Progress(
self.spinner_column,
self.test_description_column,
console=rich_console,
)
self.task = self.progress.add_task("", total=num_tests)
async def handle_post_job(token, resource: str, input_type: str = "url"):
async with httpx.AsyncClient(
headers={"authorization": f"Bearer {token}"}) as client:
with Progress(
SpinnerColumn(),
"[progress.description]{task.description}",
BarColumn(),
transient=True,
) as progress:
task = progress.add_task("Working", start=True)
job_req = await post_job(client, resource, input_type)
# job_req = await job_co
job_id = job_req.json().get("jobId")
# echo(job_req.json())
# breakpoint()
pdf = await poll_for_pdf(client, job_id, task, progress)
return pdf
def start_progress_bar(self, of=None):
"""
Starts the progress bar for the current epoch.
Args:
of: The total number of batches in the epoch or None
if that is unknown.
"""
if of is None:
if self.n_batches_last is None:
of = 0
else:
of = self.n_batches_last
if self.progress is None:
self.progress = Progress(
TextColumn("Epoch {task.fields[epoch]}"),
SpinnerColumn(),
TextColumn(
"Batch {task.fields[batch]:3} / {task.fields[of]:3}"),
BarColumn(),
TimeElapsedColumn(table_column=Column(header="Elapsed")),
TextColumn("/"),
TimeRemainingColumn(table_column=Column(header="Remaining")),
TextColumn("|"),
TextColumn("[red]Loss: {task.fields[batch_loss]:1.3f}[red],"),
TextColumn(
"[red][b]Avg.: {task.fields[running_mean]:1.3f}[/b][/red]"
),
auto_refresh=False,
transient=True,
)
fields = {
"epoch": 1,
"running_mean": np.nan,
"batch_loss": np.nan,
"of": of,
"batch": 0,
}
self.task = self.progress.add_task(f"Epoch {self.i_epoch + 1}",
total=of,
**fields)
self.progress.update(self.task,
completed=0,
total=of,
epoch=self.i_epoch + 1,
mode="training")
async def handle_post_template_jobs(token, template, data):
async with httpx.AsyncClient(
headers={"authorization": f"Bearer {token}"}) as client:
with Progress(
SpinnerColumn(),
"[progress.description]{task.description}",
BarColumn(),
transient=True,
) as progress:
jobs = []
for entry in data:
task = progress.add_task("Working", start=False)
job_req = await post_template_job(client, template, entry)
# echo(job_req)
# echo(job_req.json())
job_id = job_req.json().get("jobId")
jobs.append(poll_for_pdf(client, job_id, task, progress))
pdfs = await asyncio.gather(*jobs)
# breakpoint()
return pdfs
def get_seriler(self):
""" Sitedeki tüm animeleri [{name:*,value:*}..] formatında döndürür. """
with Progress(SpinnerColumn(),
'[progress.description]{task.description}',
BarColumn(bar_width=40)) as progress:
task = progress.add_task("[cyan]Anime listesi getiriliyor..",
start=False)
if self.tamliste:
progress.update(task, visible=False)
return self.tamliste.keys()
soup = bs4(
self.driver.execute_script(
"return $.get('https://www.turkanime.net/ajax/tamliste')"),
"html.parser")
raw_series, self.tamliste = soup.findAll(
'span', {"class": 'animeAdi'}), {}
for seri in raw_series:
self.tamliste[seri.text] = seri.findParent().get('href').split(
'anime/')[1]
progress.update(task, visible=False)
return [seri.text for seri in raw_series]
def __init__(self, num_tests: int, progress_styles: List[TestProgressStyle]):
super().__init__(num_tests, progress_styles)
self.spinner_column = SpinnerColumn(
style="pass.textonly",
finished_text=GREEN_CHECK,
)
self.bar_column = BarColumn(
complete_style="pass.textonly",
finished_style="pass.textonly",
)
self.progress: Optional[Progress] = Progress(
self.spinner_column,
TimeElapsedColumn(),
self.bar_column,
"[progress.percentage]{task.percentage:>3.0f}%",
"[progress.percentage][{task.completed} / {task.total}]",
console=self.console,
)
self.task = self.progress.add_task("Testing...", total=num_tests)
def run() -> bool:
share_progress = Progress(
"{task.description}", SpinnerColumn(), BarColumn(),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"))
accepted = share_progress.add_task("[green]Accepted",
total=len(test.functions.keys()))
rejected = share_progress.add_task("[red]Rejected",
total=len(test.functions.keys()))
total = share_progress.add_task("[blue]Total",
total=len(test.functions.keys()))
progress_table = Table.grid()
progress_table.add_row(
Panel.fit(share_progress,
title="[b]Jobs",
border_style="red",
padding=(1, 2)))
rejected_shares = []
with Live(progress_table, refresh_per_second=10):
for key in test.functions.keys():
result = test.functions[key][0](
*(test.functions[key][2] or []))
expected = test.functions[key][1]
if result == expected:
share_progress.advance(accepted)
else:
rejected_shares.append(key)
share_progress.advance(rejected)
share_progress.advance(total)