def do_multi_ping( hosts: dict, count: int = 8, interval: float = 0.01, timeout: int = 2 ) -> List[PingResult]: pool = ThreadPoolExecutor(thread_name_prefix="ping") with Progress( "[progress.description]{task.description}", BarColumn(), "{task.completed} / {task.total}", TimeRemainingColumn(), transient=True, ) as progress: jobs = [] offset = 0 for host, cc in hosts.items(): offset += 1 job = pool.submit( do_one_ping_wrapper, seq_offset=offset * len(hosts) * count, host=host, cc=cc, count=count, interval=interval, timeout=timeout, progress=progress, ) jobs.append(job) sleep(0.5) # fixme there is a deadlock in rich, need more inspect results: List[Optional[PingResult]] = list(map(lambda x: x.result(), jobs)) results: List[PingResult] = list(filter(lambda x: x is not None, results)) return results
def UploadFile(socket, address, key, size_uncompressed, size_compressed, buffer=2048): with open("temp.tar.gz", "rb") as f: file_hash_uc = SHA512.new(); file_hash_c = SHA512.new(); for address_singular in address: with open(address_singular, "rb") as filehandle: while True: block = filehandle.read(buffer); if not block: break; file_hash_uc.update(block); with Progress(TextColumn("[bold blue]{task.description}", justify="right"), BarColumn(bar_width=None), "[progress.percentage]{task.percentage:>3.1f}%", "•", DownloadColumn(), "•", TransferSpeedColumn(), "•", TimeRemainingColumn(),) as progress: task = progress.add_task("Uploading file(s)", total=size_compressed); while not progress.finished: l = f.read(buffer); if not l: break; select.select([], [socket], []); sendEncryptedMessage(socket, l, key); progress.update(task, advance=len(l)); file_hash_c.update(l); return (file_hash_uc, file_hash_c);
def put( self, path: str, key: str, acl: str = "public-read", metadata: dict = {} ) -> dict: with Progress( SpinnerColumn(spinner_name="earth"), TextColumn("[progress.description]{task.description}"), BarColumn(bar_width=30), TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), TimeRemainingColumn(), transient=True, ) as progress: size = os.stat(path).st_size task = progress.add_task( f"[green]Uploading [bold]{os.path.basename(path)}[/bold]", total=size ) def update_progress(complete): progress.update(task, completed=complete) try: response = self.client.upload_file( path, self.bucket, key, ExtraArgs={"ACL": acl, "Metadata": metadata}, Callback=update_progress, ) except ClientError as e: logging.error(e) return {} return response
def predict(trainData, trainLabel, testData, K=27): ''' 测试模型正确率 =========== Arguments --------- - `trainData` 训练集数据集 - `trainLabel` 训练集标记 - `testData` 测试集数据集 - `K` 选择近邻数 Returns ------- - `predictLabel` 预测标签 ''' predictLabel = [] progress = Progress( "[progress.description]{task.description}", BarColumn(bar_width=None), "[progress.percentage]{task.completed}/{task.total}", "•", TimeRemainingColumn(), ) # rich 进度条 progress.start() testTask = progress.add_task("[cyan]predicting...", total=len(testData)) for x in testData: predictLabel.append(NearestNeighbor(trainData, trainLabel, x, K)) # 预测标签分类 progress.update(testTask, advance=1) progress.stop() return predictLabel
def check_cdn(original_targets, checkcdn): targets = [] # 有效的目标,加上解析出的ip valid_targets = [] # 有效的目标 # 创建一个事件循环 loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) # 创建一个线程池,开启100个线程 threads = ThreadPoolExecutor(100) # 这一步很重要, 使用线程池访问,使用loop.run_in_executor()函数:内部接受的是阻塞的线程池,执行的函数,传入的参数 tasks = [] # 进度条设置 from rich.progress import ( BarColumn, TimeRemainingColumn, TransferSpeedColumn, Progress, ) progress = Progress( "[progress.description]{task.description}", BarColumn(), "[progress.percentage]{task.percentage:>3.1f}%", "•", "[bold green]{task.completed}/{task.total}", "•", TransferSpeedColumn(), "•", TimeRemainingColumn(), transient=True, # 100%后隐藏进度条 ) with progress: progress_bar = progress.add_task("[cyan]DNS, CDN detection...", total=len(original_targets)) for target in original_targets: target = target.replace('\n', '').replace('\r', '').strip() tasks.append( loop.run_in_executor(threads, run, target, checkcdn, progress_bar, progress)) if len(tasks) > 0: # 使用uvloop加速asyncio, 目前不支持Windows import platform if platform.system() != "Windows": import uvloop asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) # 等待所有的任务完成 tasks_result = asyncio.wait(tasks) loop.run_until_complete(tasks_result) for i in tasks: url_ip_list, valid_domain = i.result() targets.extend(url_ip_list) if valid_domain: valid_targets.append(valid_domain) return list(set(targets)), valid_targets
def configure_columns(self, trainer) -> list: return [ TextColumn("[progress.description]{task.description}"), BarColumn(), BatchesProcessedColumn(), TimeRemainingColumn(), TimeElapsedColumn() ]
def train_one_epoch(model, optim, loss_fn, loader, epoch, steps, device, writer, global_i, writer_interval=200, normalize=None): model.train() status_col = TextColumn("") running_loss = 0 lr = optim.param_groups[0]['lr'] if normalize is not None: assert len(normalize) == 2, "mean and std values should be provided to use data normalization" fetcher = DataPrefetcher(loader, mean=normalize[0], std=normalize[1], device=device) # modified behavior - w/ input normalization else: fetcher = DataPrefetcher(loader, mean=None, std=None, device=device) # original behavior - no input normalization samples, targets = fetcher.next() with Progress("[progress.description]{task.description}", "[{task.completed}/{task.total}]", BarColumn(), "[progress.percentage]{task.percentage:>3.0f}%", TimeRemainingColumn(), TextColumn("/"), TimeElapsedColumn(), status_col, expand=False, console=CONSOLE, refresh_per_second=5) as progress: task = progress.add_task(description=f'[Epoch {epoch}]', total=steps) i = 0 # counter t_start = time.time() while samples is not None: # zero the parameter gradients optim.zero_grad() # forward + backward + optimize out = model(samples) loss = loss_fn(out, targets) loss.backward() optim.step() # collect running loss running_loss += loss.item() i += 1 global_i += 1 # update tensorboard if i % writer_interval == 0: writer.add_scalar('Loss/Train', running_loss/i, global_i) # pre-fetch next samples samples, targets = fetcher.next() # update trackbar if not progress.finished: status_col.text_format = f"Loss: {running_loss/i:.06f} " \ f"speed: {(time.time() - t_start)/i:.4f}s/it " \ f"lr: {lr}" progress.update(task, advance=1) return running_loss / i, global_i
def _get_progress(**kwargs) -> Progress: """Default progress bar format""" return Progress( '[progress.description]{task.description}', BarColumn(), '[green]{task.completed}/{task.total}', '[progress.percentage]{task.percentage:>3.0f}%', TimeRemainingColumn(), **kwargs, )
def __init__(self, *args, **kwargs): columns = [ TextColumn("[progress.description]{task.description}"), BarColumn(), TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), DownloadColumn(), TimeRemainingColumn(), TimeElapsedColumn() ] kwargs['refresh_per_second'] = 1 super().__init__(*columns, *args, **kwargs)
def _get_progress_dl(**kwargs) -> Progress: """Track progress of processing a file in bytes""" return Progress( '[progress.description]{task.description}', BarColumn(), '[progress.percentage]{task.percentage:>3.0f}%', TransferSpeedColumn(), DownloadColumn(), TimeRemainingColumn(), **kwargs, )
def test_time_remaining_column(): class FakeTask(Task): time_remaining = 60 column = TimeRemainingColumn() task = Task(1, "test", 100, 20, _get_time=lambda: 1.0) text = column(task) assert str(text) == "-:--:--" text = column(FakeTask(1, "test", 100, 20, _get_time=lambda: 1.0)) assert str(text) == "0:01:00"
def __init__( self, n_epochs: int, train_data_len: int = 0, test_data_len: int = 0, use_info_table: bool = True, ): """ :param n_epochs: total number of epochs :param train_data_len: length of the dataset generation for training :param test_data_len: length of the dataset generation for testing :param use_info_table: true to add an information table on top of the progress bar """ self.n_epochs = n_epochs self.train_data_len = train_data_len self.test_data_len = test_data_len self.use_info_table = use_info_table self.progress = CustomProgress( TextColumn( "[bold]Epoch {task.fields[cur_epoch]}/{task.fields[n_epochs]} | [blue]{task.fields[mode]}", justify="right", ), BarColumn(bar_width=None), "[progress.percentage]{task.percentage:>3.1f}%", "•", CustomProgress.CompletedColumn(), "•", CustomProgress.TransferSpeedColumn(), "•", TimeRemainingColumn(), use_info_table=use_info_table, ) self.progress.start() self.train_p = self.progress.add_task( description="", mode="Train", cur_epoch=0, n_epochs=self.n_epochs, start=False, visible=False, total=self.train_data_len, ) self.test_p = self.progress.add_task( description="", mode="Test", cur_epoch=0, n_epochs=self.n_epochs, start=False, visible=False, total=self.test_data_len, )
def __init__(self): self.progress = Progress( TextColumn("[bold blue]{task.fields[title]}", justify="right"), BarColumn(bar_width=None), "[progress.percentage]{task.percentage:>3.1f}%", "•", DownloadColumn(), "•", TransferSpeedColumn(), "•", TimeRemainingColumn(), )
def __init__(self, infinite=False): if infinite: self.progress = Progress( "[progress.description]{task.description}", TimeElapsedColumn(), TextColumn("{task.completed} items scraped"), SpinnerColumn()) else: self.progress = Progress( "[progress.description]{task.description}", BarColumn(), "[progress.percentage]{task.percentage:>3.0f}%", TimeRemainingColumn(), "/", TimeElapsedColumn()) self.item_type_to_task = {}
def get_rich() -> Progress: return Progress( TextColumn("[bold blue]{task.description}", justify="right"), BarColumn(bar_width=None), "[progress.percentage]{task.percentage:>3.1f}%", "•", DownloadColumn(), "•", TransferSpeedColumn(), "•", TimeRemainingColumn(), )
def run(self, args): # Create a progress bar for the download progress = Progress( TextColumn("[bold cyan]{task.fields[filename]}", justify="right"), BarColumn(bar_width=None), "[progress.percentage]{task.percentage:>3.1f}%", "•", DownloadColumn(), "•", TransferSpeedColumn(), "•", TimeRemainingColumn(), ) if not args.destination: args.destination = f"./{os.path.basename(args.source)}" else: access = pwncat.victim.access(args.destination) if Access.DIRECTORY in access: args.destination = os.path.join(args.destination, os.path.basename(args.source)) elif Access.PARENT_EXIST not in access: console.log( f"[cyan]{args.destination}[/cyan]: no such file or directory" ) return try: length = os.path.getsize(args.source) started = time.time() with progress: task_id = progress.add_task("upload", filename=args.destination, total=length, start=False) with open(args.source, "rb") as source: with pwncat.victim.open(args.destination, "wb", length=length) as destination: progress.start_task(task_id) copyfileobj( source, destination, lambda count: progress.update(task_id, advance=count), ) elapsed = time.time() - started console.log(f"uploaded [cyan]{human_readable_size(length)}[/cyan] " f"in [green]{human_readable_delta(elapsed)}[/green]") except (FileNotFoundError, PermissionError, IsADirectoryError) as exc: self.parser.error(str(exc))
def __init__(self, steps: int): super().__init__() self.steps = steps self.task = None self.bar = Progress( TextColumn( "Iteration {task.fields[iteration]}/{task.fields[cumul]}"), BarColumn(), TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), TextColumn("Loss = {task.fields[loss]:.5f} | ⏳ "), TimeRemainingColumn(), TextColumn("lr = {task.fields[lr]:.6f}"), )
def create_bar(): return Progress( "[progress.percentage]{task.percentage:>3.0f}%", BarColumn(), DownloadColumn(), TextColumn("[red]["), TransferSpeedColumn(), TimeRemainingColumn(), TextColumn("[cyan]]"), TextColumn("[yellow]["), TextColumn("[yellow]{task.description}", justify="center"), TextColumn("[yellow]]"), )
def download_one_progress_bar(): """ Get progress bar to show progress for downloading a file of known size. """ progress_bar = Progress( TextColumn('[bold yellow]Downloading {task.description}'), BarColumn(), DownloadColumn(), TransferSpeedColumn(), TimeRemainingColumn(), ) return progress_bar
def track(self, tasks, label="Processing"): progress = Progress( "[progress.description][yellow]{task.description} {task.fields[key]}", BarColumn(), "[progress.percentage]{task.percentage:>3.0f}%", TimeRemainingColumn(), console=self) with progress: task_bar = progress.add_task(label, total=len(tasks), key="") for idx, task in enumerate(tasks): progress.update(task_bar, completed=idx, key=task) yield task progress.update(task_bar, advance=1, key="finished")
def predict(trainData, trainLabel, testData, kernel='Gaussian', C=200, epsilon=0.0001, sigma=10, p=2): ''' 测试模型正确率 =========== Arguments --------- - `trainData` 训练集数据集 - `trainLabel` 训练集标记 - `testData` 测试集数据集 - `kernel` 核函数 - `C` 软间隔惩罚参数 - `epsilon` 松弛变量 - `sigma` 高斯核函数参数 - `p` 多项式核参数 Returns ------- - `predictLabel` 预测标签 ''' predictLabel = [] machine = SupportVectorMachine(kernel=kernel, C=C, epsilon=epsilon, sigma=sigma) machine.train(trainData, trainLabel) progress = Progress( "[progress.description]{task.description}", BarColumn(bar_width=None), "[progress.percentage]{task.completed}/{task.total}", "•", TimeRemainingColumn(), ) # rich 进度条 progress.start() testTask = progress.add_task("[cyan]predicting...", total=len(testData)) for testDatum in testData: predictLabel.append(machine.classify(testDatum)) # 预测标签分类 progress.update(testTask, advance=1) progress.stop() return predictLabel
def _download(url: str, root: str): os.makedirs(root, exist_ok=True) filename = os.path.basename(url) download_target = os.path.join(root, filename) if os.path.isfile(download_target): return download_target if os.path.exists(download_target) and not os.path.isfile(download_target): raise FileExistsError(f'{download_target} exists and is not a regular file') from rich.progress import ( DownloadColumn, Progress, TextColumn, TimeRemainingColumn, TransferSpeedColumn, ) progress = Progress( TextColumn("[bold blue]{task.fields[filename]}", justify="right"), "[progress.percentage]{task.percentage:>3.1f}%", "•", DownloadColumn(), "•", TransferSpeedColumn(), "•", TimeRemainingColumn(), ) with progress: task = progress.add_task('download', filename=url, start=False) with urllib.request.urlopen(url) as source, open( download_target, 'wb' ) as output: progress.update(task, total=int(source.info().get('Content-Length'))) progress.start_task(task) while True: buffer = source.read(8192) if not buffer: break output.write(buffer) progress.update(task, advance=len(buffer)) return download_target
def __init__( self, description: str = 'Working...', total_length: Optional[float] = None, message_on_done: Optional[Union[str, Callable[..., str]]] = None, columns: Optional[Union[str, ProgressColumn]] = None, disable: bool = False, console: Optional[Console] = None, **kwargs, ): """Init a custom progress bar based on rich. This is the default progress bar of jina if you want to customize it you should probably just use a rich `Progress` and add your custom column and task :param description: description of your task ex : 'Working...' :param total_length: the number of steps :param message_on_done: The message that you want to print at the end of your task. It can either be a string to be formatted with task (ex '{task.completed}') task or a function which take task as input (ex : lambda task : f'{task.completed}' :param columns: If you want to customize the column of the progress bar. Note that you should probably directly use rich Progress object than overwriting these columns parameters. :param total_length: disable the progress bar .. # noqa: DAR202 .. # noqa: DAR101 .. # noqa: DAR003 """ def _default_message_on_done(task): return f'{task.completed} steps done in {get_readable_time(seconds=task.finished_time)}' columns = columns or [ SpinnerColumn(), _OnDoneColumn(f'DONE', description, 'progress.description'), BarColumn(complete_style='green', finished_style='yellow'), TimeElapsedColumn(), '[progress.percentage]{task.percentage:>3.0f}%', TextColumn('ETA:', style='progress.remaining'), TimeRemainingColumn(), _OnDoneColumn(message_on_done if message_on_done else _default_message_on_done), ] if not console: console = get_rich_console() super().__init__(*columns, console=console, disable=disable, **kwargs) self.task_id = self.add_task( 'Working...', total=total_length if total_length else 100.0)
def run(self, manager: "pwncat.manager.Manager", args): # Create a progress bar for the download progress = Progress( TextColumn("[bold cyan]{task.fields[filename]}", justify="right"), BarColumn(bar_width=None), "[progress.percentage]{task.percentage:>3.1f}%", "•", DownloadColumn(), "•", TransferSpeedColumn(), "•", TimeRemainingColumn(), ) if not args.destination: args.destination = f"./{os.path.basename(args.source)}" try: length = os.path.getsize(args.source) started = time.time() with progress: task_id = progress.add_task("upload", filename=args.destination, total=length, start=False) with open(args.source, "rb") as source: with manager.target.platform.open(args.destination, "wb") as destination: progress.start_task(task_id) copyfileobj( source, destination, lambda count: progress.update(task_id, advance=count), ) progress.update(task_id, filename="draining buffers...") progress.stop_task(task_id) progress.start_task(task_id) progress.update(task_id, filename=args.destination) elapsed = time.time() - started console.log(f"uploaded [cyan]{human_readable_size(length)}[/cyan] " f"in [green]{human_readable_delta(elapsed)}[/green]") except (FileNotFoundError, PermissionError, IsADirectoryError) as exc: self.parser.error(str(exc))
def RichTQDM(): return Progress( SpinnerColumn(), "[progress.description]{task.description}", BarColumn(), TextColumn("{task.completed}/{task.total}"), "[", TimeElapsedColumn(), "<", TimeRemainingColumn(), ',', SpeedColumn(), ']', refresh_per_second=.1, speed_estimate_period=30 )
def __init__(self, **kwargs: Any) -> None: if "transient" not in kwargs: kwargs["transient"] = True if "expand" not in kwargs: kwargs["expand"] = True super().__init__( "[progress.description]{task.description}", BarColumn(bar_width=None), "[progress.percentage]{task.percentage:>3.0f}%", TimeRemainingColumn(), **kwargs, ) self.task_description: Optional[str] = None self.task_id: Optional[TaskID] = None self.started: bool = False
def __init__(self): self.__progress = Progress( SpinnerColumn(), TextColumn( "[progress.description]{task.description} {task.fields[current]}" ), "•", BarColumn(), "•", TransferSpeedColumn(), TimeRemainingColumn(), console=Console(), transient=True, expand=True) self.__console = self.__progress.console
def parse_apk(apkfile, workers, fn_match=None, outfile=None): console = Console() console.log(f"Parsing {apkfile} with {workers} workers ...") dexes = list(extract_dex_files(apkfile)) console.log(f"Found {len(dexes)} DEX file.") total = sum(map(lambda d: len(d.data), dexes)) progress = Progress( TextColumn("[progress.description]{task.description}"), BarColumn( complete_style='bar.complete', finished_style='bar.finished', pulse_style='bar.pulse', ), TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), TimeRemainingColumn(), TimeElapsedColumn(), console=console, ) out = {} out.update(JNI_COMMON) num_classes = 0 t0 = datetime.now() with progress: task = progress.add_task("Analyzing...", total=total) with multiprocessing.Pool(workers) as pool: result = pool.imap(parse_dex_proc, dexes) for dex, count, res in result: if count == 0: console.log( "Parse {} ({} bytes) [bold red]failed: {}".format( dex.name, len(dex.data), res)) continue console.log("Parse {} ({} bytes), found {} classes.".format( dex.name, len(dex.data), count)) num_classes += count progress.update(task, advance=len(dex.data)) for cname, data in res.items(): if fn_match and not fn_match(cname): continue out.update(data) console.log("Aanlyzed {} classes, cost: {}".format(num_classes, datetime.now() - t0)) console.log("Found {} JNI methods.".format(len(out))) if not outfile: console.print_json(data=out) else: with open(outfile, 'w') as f: json.dump(out, f, indent=2, ensure_ascii=False)
def __init__(self, args: CmdArgs): self.logger = logging.getLogger('downloader') self.args = args self.exit = False # <---来自命令行的设置---> self.max_concurrent_downloads = 1 # <---进度条---> self.progress = Progress( TextColumn("[bold blue]{task.fields[name]}", justify="right"), BarColumn(bar_width=None), "[progress.percentage]{task.percentage:>3.2f}%", "•", DownloadColumn(binary_units=True), "•", TransferSpeedColumn(), "•", TimeRemainingColumn()) self.terminate = False signal.signal(signal.SIGINT, self.stop) signal.signal(signal.SIGTERM, self.stop)
def __init__(self, show_limit: int = 50): from rich.progress import Progress, BarColumn, TimeRemainingColumn self._progress = Progress( "[progress.description]{task.description}", BarColumn(style="bar.back", complete_style="bar.complete", finished_style="bar.complete"), "[progress.percentage]{task.percentage:>3.0f}%", TimeRemainingColumn(), auto_refresh=False) self._last_update = _datetime.now() self._show_limit = show_limit self._completed = {} self._have_entered = False self._enter_args = None