def do_multi_ping(
hosts: dict, count: int = 8, interval: float = 0.01, timeout: int = 2
) -> List[PingResult]:
pool = ThreadPoolExecutor(thread_name_prefix="ping")
with Progress(
"[progress.description]{task.description}",
BarColumn(),
"{task.completed} / {task.total}",
TimeRemainingColumn(),
transient=True,
) as progress:
jobs = []
offset = 0
for host, cc in hosts.items():
offset += 1
job = pool.submit(
do_one_ping_wrapper,
seq_offset=offset * len(hosts) * count,
host=host,
cc=cc,
count=count,
interval=interval,
timeout=timeout,
progress=progress,
)
jobs.append(job)
sleep(0.5) # fixme there is a deadlock in rich, need more inspect
results: List[Optional[PingResult]] = list(map(lambda x: x.result(), jobs))
results: List[PingResult] = list(filter(lambda x: x is not None, results))
return results
def UploadFile(socket, address, key, size_uncompressed, size_compressed, buffer=2048):
with open("temp.tar.gz", "rb") as f:
file_hash_uc = SHA512.new();
file_hash_c = SHA512.new();
for address_singular in address:
with open(address_singular, "rb") as filehandle:
while True:
block = filehandle.read(buffer);
if not block:
break;
file_hash_uc.update(block);
with Progress(TextColumn("[bold blue]{task.description}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),) as progress:
task = progress.add_task("Uploading file(s)", total=size_compressed);
while not progress.finished:
l = f.read(buffer);
if not l:
break;
select.select([], [socket], []);
sendEncryptedMessage(socket, l, key);
progress.update(task, advance=len(l));
file_hash_c.update(l);
return (file_hash_uc, file_hash_c);
def put(
self, path: str, key: str, acl: str = "public-read", metadata: dict = {}
) -> dict:
with Progress(
SpinnerColumn(spinner_name="earth"),
TextColumn("[progress.description]{task.description}"),
BarColumn(bar_width=30),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
TimeRemainingColumn(),
transient=True,
) as progress:
size = os.stat(path).st_size
task = progress.add_task(
f"[green]Uploading [bold]{os.path.basename(path)}[/bold]", total=size
)
def update_progress(complete):
progress.update(task, completed=complete)
try:
response = self.client.upload_file(
path,
self.bucket,
key,
ExtraArgs={"ACL": acl, "Metadata": metadata},
Callback=update_progress,
)
except ClientError as e:
logging.error(e)
return {}
return response
def predict(trainData, trainLabel, testData, K=27):
'''
测试模型正确率
===========
Arguments
---------
- `trainData` 训练集数据集
- `trainLabel` 训练集标记
- `testData` 测试集数据集
- `K` 选择近邻数
Returns
-------
- `predictLabel` 预测标签
'''
predictLabel = []
progress = Progress(
"[progress.description]{task.description}",
BarColumn(bar_width=None),
"[progress.percentage]{task.completed}/{task.total}",
"•",
TimeRemainingColumn(),
) # rich 进度条
progress.start()
testTask = progress.add_task("[cyan]predicting...", total=len(testData))
for x in testData:
predictLabel.append(NearestNeighbor(trainData, trainLabel, x,
K)) # 预测标签分类
progress.update(testTask, advance=1)
progress.stop()
return predictLabel
def check_cdn(original_targets, checkcdn):
targets = [] # 有效的目标，加上解析出的ip
valid_targets = [] # 有效的目标
# 创建一个事件循环
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
# 创建一个线程池，开启100个线程
threads = ThreadPoolExecutor(100)
# 这一步很重要, 使用线程池访问，使用loop.run_in_executor()函数:内部接受的是阻塞的线程池，执行的函数，传入的参数
tasks = []
# 进度条设置
from rich.progress import (
BarColumn,
TimeRemainingColumn,
TransferSpeedColumn,
Progress,
)
progress = Progress(
"[progress.description]{task.description}",
BarColumn(),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
"[bold green]{task.completed}/{task.total}",
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
transient=True, # 100%后隐藏进度条
)
with progress:
progress_bar = progress.add_task("[cyan]DNS, CDN detection...",
total=len(original_targets))
for target in original_targets:
target = target.replace('\n', '').replace('\r', '').strip()
tasks.append(
loop.run_in_executor(threads, run, target, checkcdn,
progress_bar, progress))
if len(tasks) > 0:
# 使用uvloop加速asyncio, 目前不支持Windows
import platform
if platform.system() != "Windows":
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
# 等待所有的任务完成
tasks_result = asyncio.wait(tasks)
loop.run_until_complete(tasks_result)
for i in tasks:
url_ip_list, valid_domain = i.result()
targets.extend(url_ip_list)
if valid_domain:
valid_targets.append(valid_domain)
return list(set(targets)), valid_targets
def configure_columns(self, trainer) -> list:
return [
TextColumn("[progress.description]{task.description}"),
BarColumn(),
BatchesProcessedColumn(),
TimeRemainingColumn(),
TimeElapsedColumn()
]
def train_one_epoch(model, optim, loss_fn, loader, epoch, steps, device,
writer, global_i, writer_interval=200, normalize=None):
model.train()
status_col = TextColumn("")
running_loss = 0
lr = optim.param_groups[0]['lr']
if normalize is not None:
assert len(normalize) == 2, "mean and std values should be provided to use data normalization"
fetcher = DataPrefetcher(loader, mean=normalize[0], std=normalize[1], device=device) # modified behavior - w/ input normalization
else:
fetcher = DataPrefetcher(loader, mean=None, std=None, device=device) # original behavior - no input normalization
samples, targets = fetcher.next()
with Progress("[progress.description]{task.description}",
"[{task.completed}/{task.total}]",
BarColumn(),
"[progress.percentage]{task.percentage:>3.0f}%",
TimeRemainingColumn(),
TextColumn("/"),
TimeElapsedColumn(),
status_col,
expand=False, console=CONSOLE, refresh_per_second=5) as progress:
task = progress.add_task(description=f'[Epoch {epoch}]', total=steps)
i = 0 # counter
t_start = time.time()
while samples is not None:
# zero the parameter gradients
optim.zero_grad()
# forward + backward + optimize
out = model(samples)
loss = loss_fn(out, targets)
loss.backward()
optim.step()
# collect running loss
running_loss += loss.item()
i += 1
global_i += 1
# update tensorboard
if i % writer_interval == 0:
writer.add_scalar('Loss/Train', running_loss/i, global_i)
# pre-fetch next samples
samples, targets = fetcher.next()
# update trackbar
if not progress.finished:
status_col.text_format = f"Loss: {running_loss/i:.06f} " \
f"speed: {(time.time() - t_start)/i:.4f}s/it " \
f"lr: {lr}"
progress.update(task, advance=1)
return running_loss / i, global_i
def _get_progress(**kwargs) -> Progress:
"""Default progress bar format"""
return Progress(
'[progress.description]{task.description}',
BarColumn(),
'[green]{task.completed}/{task.total}',
'[progress.percentage]{task.percentage:>3.0f}%',
TimeRemainingColumn(),
**kwargs,
)
def __init__(self, *args, **kwargs):
columns = [
TextColumn("[progress.description]{task.description}"),
BarColumn(),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
DownloadColumn(),
TimeRemainingColumn(),
TimeElapsedColumn()
]
kwargs['refresh_per_second'] = 1
super().__init__(*columns, *args, **kwargs)
def _get_progress_dl(**kwargs) -> Progress:
"""Track progress of processing a file in bytes"""
return Progress(
'[progress.description]{task.description}',
BarColumn(),
'[progress.percentage]{task.percentage:>3.0f}%',
TransferSpeedColumn(),
DownloadColumn(),
TimeRemainingColumn(),
**kwargs,
)
def test_time_remaining_column():
class FakeTask(Task):
time_remaining = 60
column = TimeRemainingColumn()
task = Task(1, "test", 100, 20, _get_time=lambda: 1.0)
text = column(task)
assert str(text) == "-:--:--"
text = column(FakeTask(1, "test", 100, 20, _get_time=lambda: 1.0))
assert str(text) == "0:01:00"
def __init__(
self,
n_epochs: int,
train_data_len: int = 0,
test_data_len: int = 0,
use_info_table: bool = True,
):
"""
:param n_epochs: total number of epochs
:param train_data_len: length of the dataset generation for training
:param test_data_len: length of the dataset generation for testing
:param use_info_table: true to add an information table on top of the progress bar
"""
self.n_epochs = n_epochs
self.train_data_len = train_data_len
self.test_data_len = test_data_len
self.use_info_table = use_info_table
self.progress = CustomProgress(
TextColumn(
"[bold]Epoch {task.fields[cur_epoch]}/{task.fields[n_epochs]} | [blue]{task.fields[mode]}",
justify="right",
),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
CustomProgress.CompletedColumn(),
"•",
CustomProgress.TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
use_info_table=use_info_table,
)
self.progress.start()
self.train_p = self.progress.add_task(
description="",
mode="Train",
cur_epoch=0,
n_epochs=self.n_epochs,
start=False,
visible=False,
total=self.train_data_len,
)
self.test_p = self.progress.add_task(
description="",
mode="Test",
cur_epoch=0,
n_epochs=self.n_epochs,
start=False,
visible=False,
total=self.test_data_len,
)
def __init__(self):
self.progress = Progress(
TextColumn("[bold blue]{task.fields[title]}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
)
def __init__(self, infinite=False):
if infinite:
self.progress = Progress(
"[progress.description]{task.description}",
TimeElapsedColumn(),
TextColumn("{task.completed} items scraped"), SpinnerColumn())
else:
self.progress = Progress(
"[progress.description]{task.description}", BarColumn(),
"[progress.percentage]{task.percentage:>3.0f}%",
TimeRemainingColumn(), "/", TimeElapsedColumn())
self.item_type_to_task = {}
def get_rich() -> Progress:
return Progress(
TextColumn("[bold blue]{task.description}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
)
def run(self, args):
# Create a progress bar for the download
progress = Progress(
TextColumn("[bold cyan]{task.fields[filename]}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
)
if not args.destination:
args.destination = f"./{os.path.basename(args.source)}"
else:
access = pwncat.victim.access(args.destination)
if Access.DIRECTORY in access:
args.destination = os.path.join(args.destination,
os.path.basename(args.source))
elif Access.PARENT_EXIST not in access:
console.log(
f"[cyan]{args.destination}[/cyan]: no such file or directory"
)
return
try:
length = os.path.getsize(args.source)
started = time.time()
with progress:
task_id = progress.add_task("upload",
filename=args.destination,
total=length,
start=False)
with open(args.source, "rb") as source:
with pwncat.victim.open(args.destination,
"wb",
length=length) as destination:
progress.start_task(task_id)
copyfileobj(
source,
destination,
lambda count: progress.update(task_id,
advance=count),
)
elapsed = time.time() - started
console.log(f"uploaded [cyan]{human_readable_size(length)}[/cyan] "
f"in [green]{human_readable_delta(elapsed)}[/green]")
except (FileNotFoundError, PermissionError, IsADirectoryError) as exc:
self.parser.error(str(exc))
def __init__(self, steps: int):
super().__init__()
self.steps = steps
self.task = None
self.bar = Progress(
TextColumn(
"Iteration {task.fields[iteration]}/{task.fields[cumul]}"),
BarColumn(),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
TextColumn("Loss = {task.fields[loss]:.5f} | ⏳ "),
TimeRemainingColumn(),
TextColumn("lr = {task.fields[lr]:.6f}"),
)
def create_bar():
return Progress(
"[progress.percentage]{task.percentage:>3.0f}%",
BarColumn(),
DownloadColumn(),
TextColumn("[red]["),
TransferSpeedColumn(),
TimeRemainingColumn(),
TextColumn("[cyan]]"),
TextColumn("[yellow]["),
TextColumn("[yellow]{task.description}", justify="center"),
TextColumn("[yellow]]"),
)
def download_one_progress_bar():
"""
Get progress bar to show progress for downloading a file of known size.
"""
progress_bar = Progress(
TextColumn('[bold yellow]Downloading {task.description}'),
BarColumn(),
DownloadColumn(),
TransferSpeedColumn(),
TimeRemainingColumn(),
)
return progress_bar
def track(self, tasks, label="Processing"):
progress = Progress(
"[progress.description][yellow]{task.description} {task.fields[key]}",
BarColumn(),
"[progress.percentage]{task.percentage:>3.0f}%",
TimeRemainingColumn(),
console=self)
with progress:
task_bar = progress.add_task(label, total=len(tasks), key="")
for idx, task in enumerate(tasks):
progress.update(task_bar, completed=idx, key=task)
yield task
progress.update(task_bar, advance=1, key="finished")
def predict(trainData,
trainLabel,
testData,
kernel='Gaussian',
C=200,
epsilon=0.0001,
sigma=10,
p=2):
'''
测试模型正确率
===========
Arguments
---------
- `trainData` 训练集数据集
- `trainLabel` 训练集标记
- `testData` 测试集数据集
- `kernel` 核函数
- `C` 软间隔惩罚参数
- `epsilon` 松弛变量
- `sigma` 高斯核函数参数
- `p` 多项式核参数
Returns
-------
- `predictLabel` 预测标签
'''
predictLabel = []
machine = SupportVectorMachine(kernel=kernel,
C=C,
epsilon=epsilon,
sigma=sigma)
machine.train(trainData, trainLabel)
progress = Progress(
"[progress.description]{task.description}",
BarColumn(bar_width=None),
"[progress.percentage]{task.completed}/{task.total}",
"•",
TimeRemainingColumn(),
) # rich 进度条
progress.start()
testTask = progress.add_task("[cyan]predicting...", total=len(testData))
for testDatum in testData:
predictLabel.append(machine.classify(testDatum)) # 预测标签分类
progress.update(testTask, advance=1)
progress.stop()
return predictLabel
def _download(url: str, root: str):
os.makedirs(root, exist_ok=True)
filename = os.path.basename(url)
download_target = os.path.join(root, filename)
if os.path.isfile(download_target):
return download_target
if os.path.exists(download_target) and not os.path.isfile(download_target):
raise FileExistsError(f'{download_target} exists and is not a regular file')
from rich.progress import (
DownloadColumn,
Progress,
TextColumn,
TimeRemainingColumn,
TransferSpeedColumn,
)
progress = Progress(
TextColumn("[bold blue]{task.fields[filename]}", justify="right"),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
)
with progress:
task = progress.add_task('download', filename=url, start=False)
with urllib.request.urlopen(url) as source, open(
download_target, 'wb'
) as output:
progress.update(task, total=int(source.info().get('Content-Length')))
progress.start_task(task)
while True:
buffer = source.read(8192)
if not buffer:
break
output.write(buffer)
progress.update(task, advance=len(buffer))
return download_target
def __init__(
self,
description: str = 'Working...',
total_length: Optional[float] = None,
message_on_done: Optional[Union[str, Callable[..., str]]] = None,
columns: Optional[Union[str, ProgressColumn]] = None,
disable: bool = False,
console: Optional[Console] = None,
**kwargs,
):
"""Init a custom progress bar based on rich. This is the default progress bar of jina if you want to customize
it you should probably just use a rich `Progress` and add your custom column and task
:param description: description of your task ex : 'Working...'
:param total_length: the number of steps
:param message_on_done: The message that you want to print at the end of your task. It can either be a string to
be formatted with task (ex '{task.completed}') task or a function which take task as input (ex : lambda task : f'{task.completed}'
:param columns: If you want to customize the column of the progress bar. Note that you should probably directly use
rich Progress object than overwriting these columns parameters.
:param total_length: disable the progress bar
.. # noqa: DAR202
.. # noqa: DAR101
.. # noqa: DAR003
"""
def _default_message_on_done(task):
return f'{task.completed} steps done in {get_readable_time(seconds=task.finished_time)}'
columns = columns or [
SpinnerColumn(),
_OnDoneColumn(f'DONE', description, 'progress.description'),
BarColumn(complete_style='green', finished_style='yellow'),
TimeElapsedColumn(),
'[progress.percentage]{task.percentage:>3.0f}%',
TextColumn('ETA:', style='progress.remaining'),
TimeRemainingColumn(),
_OnDoneColumn(message_on_done
if message_on_done else _default_message_on_done),
]
if not console:
console = get_rich_console()
super().__init__(*columns, console=console, disable=disable, **kwargs)
self.task_id = self.add_task(
'Working...', total=total_length if total_length else 100.0)
def run(self, manager: "pwncat.manager.Manager", args):
# Create a progress bar for the download
progress = Progress(
TextColumn("[bold cyan]{task.fields[filename]}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
)
if not args.destination:
args.destination = f"./{os.path.basename(args.source)}"
try:
length = os.path.getsize(args.source)
started = time.time()
with progress:
task_id = progress.add_task("upload",
filename=args.destination,
total=length,
start=False)
with open(args.source, "rb") as source:
with manager.target.platform.open(args.destination,
"wb") as destination:
progress.start_task(task_id)
copyfileobj(
source,
destination,
lambda count: progress.update(task_id,
advance=count),
)
progress.update(task_id,
filename="draining buffers...")
progress.stop_task(task_id)
progress.start_task(task_id)
progress.update(task_id, filename=args.destination)
elapsed = time.time() - started
console.log(f"uploaded [cyan]{human_readable_size(length)}[/cyan] "
f"in [green]{human_readable_delta(elapsed)}[/green]")
except (FileNotFoundError, PermissionError, IsADirectoryError) as exc:
self.parser.error(str(exc))
def RichTQDM():
return Progress(
SpinnerColumn(),
"[progress.description]{task.description}",
BarColumn(),
TextColumn("{task.completed}/{task.total}"),
"[",
TimeElapsedColumn(),
"<",
TimeRemainingColumn(),
',',
SpeedColumn(),
']',
refresh_per_second=.1, speed_estimate_period=30
)
def __init__(self, **kwargs: Any) -> None:
if "transient" not in kwargs:
kwargs["transient"] = True
if "expand" not in kwargs:
kwargs["expand"] = True
super().__init__(
"[progress.description]{task.description}",
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.0f}%",
TimeRemainingColumn(),
**kwargs,
)
self.task_description: Optional[str] = None
self.task_id: Optional[TaskID] = None
self.started: bool = False
def __init__(self):
self.__progress = Progress(
SpinnerColumn(),
TextColumn(
"[progress.description]{task.description} {task.fields[current]}"
),
"•",
BarColumn(),
"•",
TransferSpeedColumn(),
TimeRemainingColumn(),
console=Console(),
transient=True,
expand=True)
self.__console = self.__progress.console
def parse_apk(apkfile, workers, fn_match=None, outfile=None):
console = Console()
console.log(f"Parsing {apkfile} with {workers} workers ...")
dexes = list(extract_dex_files(apkfile))
console.log(f"Found {len(dexes)} DEX file.")
total = sum(map(lambda d: len(d.data), dexes))
progress = Progress(
TextColumn("[progress.description]{task.description}"),
BarColumn(
complete_style='bar.complete',
finished_style='bar.finished',
pulse_style='bar.pulse',
),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
TimeRemainingColumn(),
TimeElapsedColumn(),
console=console,
)
out = {}
out.update(JNI_COMMON)
num_classes = 0
t0 = datetime.now()
with progress:
task = progress.add_task("Analyzing...", total=total)
with multiprocessing.Pool(workers) as pool:
result = pool.imap(parse_dex_proc, dexes)
for dex, count, res in result:
if count == 0:
console.log(
"Parse {} ({} bytes) [bold red]failed: {}".format(
dex.name, len(dex.data), res))
continue
console.log("Parse {} ({} bytes), found {} classes.".format(
dex.name, len(dex.data), count))
num_classes += count
progress.update(task, advance=len(dex.data))
for cname, data in res.items():
if fn_match and not fn_match(cname):
continue
out.update(data)
console.log("Aanlyzed {} classes, cost: {}".format(num_classes,
datetime.now() - t0))
console.log("Found {} JNI methods.".format(len(out)))
if not outfile:
console.print_json(data=out)
else:
with open(outfile, 'w') as f:
json.dump(out, f, indent=2, ensure_ascii=False)
def __init__(self, args: CmdArgs):
self.logger = logging.getLogger('downloader')
self.args = args
self.exit = False
# <---来自命令行的设置--->
self.max_concurrent_downloads = 1
# <---进度条--->
self.progress = Progress(
TextColumn("[bold blue]{task.fields[name]}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.2f}%", "•",
DownloadColumn(binary_units=True), "•", TransferSpeedColumn(), "•",
TimeRemainingColumn())
self.terminate = False
signal.signal(signal.SIGINT, self.stop)
signal.signal(signal.SIGTERM, self.stop)
def __init__(self, show_limit: int = 50):
from rich.progress import Progress, BarColumn, TimeRemainingColumn
self._progress = Progress(
"[progress.description]{task.description}",
BarColumn(style="bar.back",
complete_style="bar.complete",
finished_style="bar.complete"),
"[progress.percentage]{task.percentage:>3.0f}%",
TimeRemainingColumn(),
auto_refresh=False)
self._last_update = _datetime.now()
self._show_limit = show_limit
self._completed = {}
self._have_entered = False
self._enter_args = None
