def check_cdn(original_targets, checkcdn):
targets = [] # 有效的目标，加上解析出的ip
valid_targets = [] # 有效的目标
# 创建一个事件循环
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
# 创建一个线程池，开启100个线程
threads = ThreadPoolExecutor(100)
# 这一步很重要, 使用线程池访问，使用loop.run_in_executor()函数:内部接受的是阻塞的线程池，执行的函数，传入的参数
tasks = []
# 进度条设置
from rich.progress import (
BarColumn,
TimeRemainingColumn,
TransferSpeedColumn,
Progress,
)
progress = Progress(
"[progress.description]{task.description}",
BarColumn(),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
"[bold green]{task.completed}/{task.total}",
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
transient=True, # 100%后隐藏进度条
)
with progress:
progress_bar = progress.add_task("[cyan]DNS, CDN detection...",
total=len(original_targets))
for target in original_targets:
target = target.replace('\n', '').replace('\r', '').strip()
tasks.append(
loop.run_in_executor(threads, run, target, checkcdn,
progress_bar, progress))
if len(tasks) > 0:
# 使用uvloop加速asyncio, 目前不支持Windows
import platform
if platform.system() != "Windows":
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
# 等待所有的任务完成
tasks_result = asyncio.wait(tasks)
loop.run_until_complete(tasks_result)
for i in tasks:
url_ip_list, valid_domain = i.result()
targets.extend(url_ip_list)
if valid_domain:
valid_targets.append(valid_domain)
return list(set(targets)), valid_targets
def _get_progress_dl(**kwargs) -> Progress:
"""Track progress of processing a file in bytes"""
return Progress(
'[progress.description]{task.description}',
BarColumn(),
'[progress.percentage]{task.percentage:>3.0f}%',
TransferSpeedColumn(),
DownloadColumn(),
TimeRemainingColumn(),
**kwargs,
)
def __init__(self, client_id, client_secret):
self.api = Api(client_id, client_secret)
self.progress = Progress(
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
DownloadColumn(),
TransferSpeedColumn(),
"[bold blue]{task.fields[filename]}",
)
self.all_t = self.progress.add_task('All', filename='All', start=0)
self.total_length = 0;
def download_one_progress_bar_unknown_size():
"""
Get progress bar to show progress for downloading a file of unknown size.
"""
progress_bar = Progress(
TextColumn('[bold yellow]Downloading {task.description}'),
FileSizeColumn(),
TransferSpeedColumn(),
)
return progress_bar
def __init__(self):
self.progress = Progress(
TextColumn("[bold blue]{task.fields[title]}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
)
def get_rich() -> Progress:
return Progress(
TextColumn("[bold blue]{task.description}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
)
def run(self, args):
# Create a progress bar for the download
progress = Progress(
TextColumn("[bold cyan]{task.fields[filename]}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
)
if not args.destination:
args.destination = f"./{os.path.basename(args.source)}"
else:
access = pwncat.victim.access(args.destination)
if Access.DIRECTORY in access:
args.destination = os.path.join(args.destination,
os.path.basename(args.source))
elif Access.PARENT_EXIST not in access:
console.log(
f"[cyan]{args.destination}[/cyan]: no such file or directory"
)
return
try:
length = os.path.getsize(args.source)
started = time.time()
with progress:
task_id = progress.add_task("upload",
filename=args.destination,
total=length,
start=False)
with open(args.source, "rb") as source:
with pwncat.victim.open(args.destination,
"wb",
length=length) as destination:
progress.start_task(task_id)
copyfileobj(
source,
destination,
lambda count: progress.update(task_id,
advance=count),
)
elapsed = time.time() - started
console.log(f"uploaded [cyan]{human_readable_size(length)}[/cyan] "
f"in [green]{human_readable_delta(elapsed)}[/green]")
except (FileNotFoundError, PermissionError, IsADirectoryError) as exc:
self.parser.error(str(exc))
def create_bar():
return Progress(
"[progress.percentage]{task.percentage:>3.0f}%",
BarColumn(),
DownloadColumn(),
TextColumn("[red]["),
TransferSpeedColumn(),
TimeRemainingColumn(),
TextColumn("[cyan]]"),
TextColumn("[yellow]["),
TextColumn("[yellow]{task.description}", justify="center"),
TextColumn("[yellow]]"),
)
def download_one_progress_bar():
"""
Get progress bar to show progress for downloading a file of known size.
"""
progress_bar = Progress(
TextColumn('[bold yellow]Downloading {task.description}'),
BarColumn(),
DownloadColumn(),
TransferSpeedColumn(),
TimeRemainingColumn(),
)
return progress_bar
def _download(url: str, root: str):
os.makedirs(root, exist_ok=True)
filename = os.path.basename(url)
download_target = os.path.join(root, filename)
if os.path.isfile(download_target):
return download_target
if os.path.exists(download_target) and not os.path.isfile(download_target):
raise FileExistsError(f'{download_target} exists and is not a regular file')
from rich.progress import (
DownloadColumn,
Progress,
TextColumn,
TimeRemainingColumn,
TransferSpeedColumn,
)
progress = Progress(
TextColumn("[bold blue]{task.fields[filename]}", justify="right"),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
)
with progress:
task = progress.add_task('download', filename=url, start=False)
with urllib.request.urlopen(url) as source, open(
download_target, 'wb'
) as output:
progress.update(task, total=int(source.info().get('Content-Length')))
progress.start_task(task)
while True:
buffer = source.read(8192)
if not buffer:
break
output.write(buffer)
progress.update(task, advance=len(buffer))
return download_target
def run(self, manager: "pwncat.manager.Manager", args):
# Create a progress bar for the download
progress = Progress(
TextColumn("[bold cyan]{task.fields[filename]}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
)
if not args.destination:
args.destination = f"./{os.path.basename(args.source)}"
try:
length = os.path.getsize(args.source)
started = time.time()
with progress:
task_id = progress.add_task("upload",
filename=args.destination,
total=length,
start=False)
with open(args.source, "rb") as source:
with manager.target.platform.open(args.destination,
"wb") as destination:
progress.start_task(task_id)
copyfileobj(
source,
destination,
lambda count: progress.update(task_id,
advance=count),
)
progress.update(task_id,
filename="draining buffers...")
progress.stop_task(task_id)
progress.start_task(task_id)
progress.update(task_id, filename=args.destination)
elapsed = time.time() - started
console.log(f"uploaded [cyan]{human_readable_size(length)}[/cyan] "
f"in [green]{human_readable_delta(elapsed)}[/green]")
except (FileNotFoundError, PermissionError, IsADirectoryError) as exc:
self.parser.error(str(exc))
def __init__(self):
self.__progress = Progress(
SpinnerColumn(),
TextColumn(
"[progress.description]{task.description} {task.fields[current]}"
),
"•",
BarColumn(),
"•",
TransferSpeedColumn(),
TimeRemainingColumn(),
console=Console(),
transient=True,
expand=True)
self.__console = self.__progress.console
def __init__(self, args: CmdArgs):
self.logger = logging.getLogger('downloader')
self.args = args
self.exit = False
# <---来自命令行的设置--->
self.max_concurrent_downloads = 1
# <---进度条--->
self.progress = Progress(
TextColumn("[bold blue]{task.fields[name]}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.2f}%", "•",
DownloadColumn(binary_units=True), "•", TransferSpeedColumn(), "•",
TimeRemainingColumn())
self.terminate = False
signal.signal(signal.SIGINT, self.stop)
signal.signal(signal.SIGTERM, self.stop)
def __init__(self):
self.exit = False
# <---来自命令行的设置--->
self.max_concurrent_downloads = 1
# <---进度条--->
self.progress = Progress(
TextColumn("[bold blue]{task.fields[name]}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.2f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
)
def run(self, cfg: DictConfig):
if not cfg.task.query:
raise ValueError("Query must be provided.")
driver, aq = self.sessions(cfg)
with Progress(
"[progress.description]{task.description}",
BarColumn(),
"[progress.percentage]{task.percentage:>3.0f}%",
TimeRemainingColumn(),
TransferSpeedColumn(),
DownloadColumn(),
) as progress:
n_cpus = cfg.task.n_jobs or os.cpu_count()
# TASK 0
logger.info("Requesting Aquarium inventory...")
match_query = "MATCH (n:Sample) RETURN n.id"
if cfg.task.query.n_items > -1:
match_query += " LIMIT {}".format(cfg.task.query.n_items)
results = driver.read(match_query)
sample_ids = [r[0] for r in results]
models = aq.Sample.find(sample_ids)
logger.info("Found {} samples in graph db".format(len(models)))
node_payload = aq_inventory_to_cypher(aq, models)
if cfg.task.strict:
def error_callback(e: Exception):
raise e
else:
error_callback = self.catch_constraint_error
task1 = progress.add_task("adding nodes...")
# TODO: indicate when creation is skipped
if cfg.task.create_nodes:
progress.tasks[task1].total = len(node_payload)
driver.pool(n_cpus).write(
node_payload,
callback=lambda _: progress.update(task1, advance=1),
chunksize=cfg.task.chunksize,
error_callback=error_callback,
)
progress.update(task1, completed=progress.tasks[task1].total)
def test_columns() -> None:
console = Console(
file=io.StringIO(),
force_terminal=True,
width=80,
log_time_format="[TIME]",
color_system="truecolor",
legacy_windows=False,
log_path=False,
_environ={},
)
progress = Progress(
"test",
TextColumn("{task.description}"),
BarColumn(bar_width=None),
TimeRemainingColumn(),
TimeElapsedColumn(),
FileSizeColumn(),
TotalFileSizeColumn(),
DownloadColumn(),
TransferSpeedColumn(),
MofNCompleteColumn(),
MofNCompleteColumn(separator=" of "),
transient=True,
console=console,
auto_refresh=False,
get_time=MockClock(),
)
task1 = progress.add_task("foo", total=10)
task2 = progress.add_task("bar", total=7)
with progress:
for n in range(4):
progress.advance(task1, 3)
progress.advance(task2, 4)
print("foo")
console.log("hello")
console.print("world")
progress.refresh()
from .render import replace_link_ids
result = replace_link_ids(console.file.getvalue())
print(repr(result))
expected = "\x1b[?25ltest foo \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:07\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \x1b[32m 0/10\x1b[0m \x1b[32m 0 of 10\x1b[0m\ntest bar \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:18\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \x1b[32m0/7 \x1b[0m \x1b[32m0 of 7 \x1b[0m\r\x1b[2K\x1b[1A\x1b[2Kfoo\ntest foo \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:07\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \x1b[32m 0/10\x1b[0m \x1b[32m 0 of 10\x1b[0m\ntest bar \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:18\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \x1b[32m0/7 \x1b[0m \x1b[32m0 of 7 \x1b[0m\r\x1b[2K\x1b[1A\x1b[2K\x1b[2;36m[TIME]\x1b[0m\x1b[2;36m \x1b[0mhello \ntest foo \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:07\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \x1b[32m 0/10\x1b[0m \x1b[32m 0 of 10\x1b[0m\ntest bar \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:18\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \x1b[32m0/7 \x1b[0m \x1b[32m0 of 7 \x1b[0m\r\x1b[2K\x1b[1A\x1b[2Kworld\ntest foo \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:07\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \x1b[32m 0/10\x1b[0m \x1b[32m 0 of 10\x1b[0m\ntest bar \x1b[38;5;237m━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[33m0:00:18\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \x1b[32m0/7 \x1b[0m \x1b[32m0 of 7 \x1b[0m\r\x1b[2K\x1b[1A\x1b[2Ktest foo \x1b[38;2;114;156;31m━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[33m0:00:34\x1b[0m \x1b[32m12 \x1b[0m \x1b[32m10 \x1b[0m \x1b[32m12/10 \x1b[0m \x1b[31m1 \x1b[0m \x1b[32m12/10\x1b[0m \x1b[32m12 of 10\x1b[0m\n \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[31mbyte/s \x1b[0m \ntest bar \x1b[38;2;114;156;31m━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[33m0:00:29\x1b[0m \x1b[32m16 \x1b[0m \x1b[32m7 bytes\x1b[0m \x1b[32m16/7 \x1b[0m \x1b[31m2 \x1b[0m \x1b[32m16/7 \x1b[0m \x1b[32m16 of 7 \x1b[0m\n \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[31mbytes/s\x1b[0m \r\x1b[2K\x1b[1A\x1b[2K\x1b[1A\x1b[2K\x1b[1A\x1b[2Ktest foo \x1b[38;2;114;156;31m━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[33m0:00:34\x1b[0m \x1b[32m12 \x1b[0m \x1b[32m10 \x1b[0m \x1b[32m12/10 \x1b[0m \x1b[31m1 \x1b[0m \x1b[32m12/10\x1b[0m \x1b[32m12 of 10\x1b[0m\n \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[31mbyte/s \x1b[0m \ntest bar \x1b[38;2;114;156;31m━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[33m0:00:29\x1b[0m \x1b[32m16 \x1b[0m \x1b[32m7 bytes\x1b[0m \x1b[32m16/7 \x1b[0m \x1b[31m2 \x1b[0m \x1b[32m16/7 \x1b[0m \x1b[32m16 of 7 \x1b[0m\n \x1b[32mbytes \x1b[0m \x1b[32mbytes \x1b[0m \x1b[31mbytes/s\x1b[0m \n\x1b[?25h\r\x1b[1A\x1b[2K\x1b[1A\x1b[2K\x1b[1A\x1b[2K\x1b[1A\x1b[2K"
assert result == expected
def retrieve_over_http(url, output_file_path):
"""Download file from remote location, with progress bar.
Parameters
----------
url : str
Remote URL.
output_file_path : str or Path
Full file destination for download.
"""
# Make Rich progress bar
progress = Progress(
TextColumn("[bold]Downloading...", justify="right"),
BarColumn(bar_width=None),
"{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"• speed:",
TransferSpeedColumn(),
"• ETA:",
TimeRemainingColumn(),
)
CHUNK_SIZE = 4096
response = requests.get(url, stream=True)
try:
with progress:
task_id = progress.add_task(
"download",
filename=output_file_path.name,
start=True,
total=int(response.headers.get("content-length", 0)),
)
with open(output_file_path, "wb") as fout:
for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
fout.write(chunk)
progress.update(task_id, advance=len(chunk), refresh=True)
except requests.exceptions.ConnectionError:
output_file_path.unlink()
raise requests.exceptions.ConnectionError(
f"Could not download file from {url}"
)
def DownloadFile(socket, name, key, size_uncompressed, size_compressed, buffer=2048):
with open("temp.tar.gz", "wb") as f:
file_hash = SHA512.new();
with Progress(TextColumn("[bold blue]{task.description}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),) as progress:
task = progress.add_task(f"Downloading file(s)", total=size_compressed);
while not progress.finished:
select.select([client_socket], [], []);
user_data = receive_message(socket);
l = recieveEncryptedMessage(socket, key)["data"];
f.write(l);
progress.update(task, advance=len(l));
file_hash.update(l);
user_data = receive_message(socket);
l = recieveEncryptedMessage(socket, key)["data"];
if l[:8] == "SFTP END".encode('utf-8'):
print(f"{RT.BLUE}SFTP END{RT.RESET}");
else:
print(f"{RT.RED}SFTP Did Not End! Retry File Transfer.{RT.End}");
return;
split_data = l.split(CUSTOM_SEPARATOR);
received_file_hash_uc = split_data[1].decode('utf-8');
received_file_hash_c = split_data[2].decode('utf-8');
if received_file_hash_c == file_hash.hexdigest():
FileDecompressor("temp.tar.gz", name);
ucfilehash = SHA512.new();
for name_singular in name:
with open(name_singular, "rb") as filehandle:
while True:
block = filehandle.read(buffer);
if not block:
break;
ucfilehash.update(block);
if received_file_hash_c == file_hash.hexdigest() and received_file_hash_uc == ucfilehash.hexdigest():
print(f"{RT.GREEN}SFTP Checksum Matched!{RT.RESET}");
else:
print(f"{RT.RED}SFTP Checksum Did Not Match! File Is Corrupt{RT.RESET}");
def transfer(source: Path, dest: Path):
"""
Transfers the directory `source` into the existing directory `dest`.
Unlike rsync, we walk the entire source at the start and sum up the total size we
will be transferring. If there's lots of files this can take awhile!
We don't have anyway of knowing if some of the source is already at the
destination (for example, if there is a previously-interrupted rsync operation).
If there is already some of the source data there, the progress bar will not be
accurate until the very end where rsync tells us the operation is complete.
"""
source_size = get_file_sizes(source)
with Progress(
"{task.description}",
BarColumn(),
"[progress.percentage]{task.percentage:>3.0f}%",
TimeRemainingColumn(),
TransferSpeedColumn(),
FileSizeColumn(),
"/",
TotalFileSizeColumn(),
auto_refresh=False,
) as progress:
transfer_task = progress.add_task("Transferring...", total=source_size)
for stats in rsyncwrap(source, dest, include_raw_output=True):
if not isinstance(stats, int):
progress.update(transfer_task,
completed=stats.total_transferred)
progress.refresh()
# Since rsync doesn't necessarily transfer every byte when destination files
# already exist, we finally update progress...as long as there wasn't a
# problem return code from rsync.
if stats == 0:
# TODO: This last update seems to throw the transfer rate figure way off?
progress.update(transfer_task, completed=source_size)
progress.refresh()
assert stats == 0, f"Received error return code from rsync: {stats}"
def test_columns() -> None:
time = 1.0
def get_time():
nonlocal time
time += 1.0
return time
console = Console(
file=io.StringIO(),
force_terminal=True,
width=80,
log_time_format="[TIME]",
color_system="truecolor",
)
progress = Progress(
"test",
TextColumn("{task.description}"),
BarColumn(bar_width=None),
TimeRemainingColumn(),
FileSizeColumn(),
TotalFileSizeColumn(),
DownloadColumn(),
TransferSpeedColumn(),
console=console,
auto_refresh=False,
get_time=get_time,
)
task1 = progress.add_task("foo", total=10)
task2 = progress.add_task("bar", total=7)
with progress:
for n in range(4):
progress.advance(task1, 3)
progress.advance(task2, 4)
progress.log("hello")
progress.print("world")
progress.refresh()
result = console.file.getvalue()
print(repr(result))
expected = 'test foo \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \r\x1b[2Ktest foo \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \ntest bar \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \x1b[?25l\r\x1b[1A\x1b[2Ktest foo \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \ntest bar \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \r\x1b[1A\x1b[2K\x1b[2;36m[TIME]\x1b[0m\x1b[2;36m \x1b[0mhello \x1b[2mtest_progress.py:190\x1b[0m\x1b[2m \x1b[0m\ntest foo \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \ntest bar \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \r\x1b[1A\x1b[2Kworld\ntest foo \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m0/10 bytes\x1b[0m \x1b[31m?\x1b[0m \ntest bar \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[36m-:--:--\x1b[0m \x1b[32m0 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m0/7 bytes \x1b[0m \x1b[31m?\x1b[0m \r\x1b[1A\x1b[2Ktest foo \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[32m12 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m12/10 bytes\x1b[0m \x1b[31m1 byte/s \x1b[0m \ntest bar \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[32m16 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m16/7 bytes \x1b[0m \x1b[31m2 bytes/s\x1b[0m \r\x1b[1A\x1b[2Ktest foo \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[32m12 bytes\x1b[0m \x1b[32m10 bytes\x1b[0m \x1b[32m12/10 bytes\x1b[0m \x1b[31m1 byte/s \x1b[0m \ntest bar \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[36m0:00:00\x1b[0m \x1b[32m16 bytes\x1b[0m \x1b[32m7 bytes \x1b[0m \x1b[32m16/7 bytes \x1b[0m \x1b[31m2 bytes/s\x1b[0m \n\x1b[?25h'
assert result == expected
def __init__(self, targets, rate=2000, timeout=3):
super(PortScan, self).__init__()
self.targets = targets
self.hosts = []
self.rate = rate # 限制并发量
self.timeout = timeout
self.result = []
self.process = Progress(
"[progress.description]{task.description}",
BarColumn(),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
"[bold green]{task.completed}/{task.total}",
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
transient=True, # 100%后隐藏进度条
)
self.progress_bar = self.process.add_task("[cyan]port scan...",
total=len(self.targets))
def __init__(self, targets, fofa_result):
super(Fofa, self).__init__()
self.email = fofaApi['email']
self.key = fofaApi['key']
self.fofa_result = fofa_result
self.targets = targets
self.result_urls = [] # fofa 查询到的web服务列表
self.urls_list = [] # 去重
self.life_urls = [] # 验证存活的web服务列表
self.urls = [] # fofa查询的 url 列表, 供异步协程使用
self.count = 30 # fofa 一次性查多少个
self.session = conn_pool() # 使用连接池
self.headers = {
"Cache-Control":
"max-age=0",
"User-Agent":
random.choice(USER_AGENTS),
"Upgrade-Insecure-Requests":
"1",
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
}
self.process = Progress(
"[progress.description]{task.description}",
BarColumn(),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
"[bold green]{task.completed}/{task.total}",
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
transient=True, # 100%后隐藏进度条
)
self.fofa_progress_bar = self.process.add_task("[cyan]FOFA search...",
total=len(self.targets))
self.web_progress_bar = None
async def main(url=None, file=None, task_count=None):
if url is None:
url = input('url: ')
if file is None:
file = input('file: ')
if task_count is None:
task_count = int(input('task count (default 10): ').strip() or 10)
progress = Progress(
TextColumn('[bold blue]{task.description}', justify='right'),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
refresh_per_second=1
)
with open(file, 'wb') as fp, progress:
async with Downloader(url, fp, {}, task_count) as downloader:
t_tasks = [progress.add_task('Total', total=downloader.filesize)]
bytes_last_task = (downloader.filesize -
downloader.bytes_per_task * (downloader.task_count - 1))
for i in range(1, downloader.task_count + 1):
t_tasks.append(progress.add_task(
f'{i}',
total=downloader.bytes_per_task
if i != downloader.task_count
else bytes_last_task
))
def report_hook(task_id, task_downloaded, total_downloaded):
progress.update(t_tasks[task_id], completed=task_downloaded)
progress.update(t_tasks[0], completed=total_downloaded)
await downloader.download(report_hook)
input('Finished... ')
def DataTransformBar(has_size_info: bool = True):
"""
创建一个数据传输进度条
Create a data transfer progress bar
:param has_size_info: 是否有总任务量
:return: rich.progress.Progress
"""
from rich.progress import Progress, TextColumn, BarColumn, DownloadColumn, TransferSpeedColumn, TimeRemainingColumn
from .. import qs_default_console
if has_size_info:
return Progress(
TextColumn("[bold blue]{task.fields[filename]}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
console=qs_default_console
)
else:
from .. import user_lang
return Progress(
TextColumn(
"[bold blue]{task.fields[filename]} [red]" +
('Unknow size' if user_lang != 'zh' else '未知大小'),
justify="right"
),
BarColumn(bar_width=None),
DownloadColumn(),
console=qs_default_console
)
class RichTransferProgress(RichProgress):
SUMMARY_COLS = (
TextColumn("[magenta]{task.description}[bold green]"),
MofNCompleteColumnWithUnit(),
TimeElapsedColumn(),
)
TRANSFER_COLS = (
TextColumn(" [blue]{task.description}"),
BarColumn(),
DownloadColumn(),
TransferSpeedColumn(),
TextColumn("eta"),
TimeRemainingColumn(),
)
def get_renderables(self):
summary_tasks, other_tasks = split(
lambda task: task.fields.get("progress_type") == "summary",
self.tasks,
)
self.columns = self.SUMMARY_COLS
yield self.make_tasks_table(summary_tasks)
self.columns = self.TRANSFER_COLS
yield self.make_tasks_table(other_tasks)
def run(self, cfg: Optional[DictConfig] = None):
"""Update the graphdb using some Aquarium sample query.
:param cfg: the configuration
:return: None
"""
driver, aq = self.sessions(cfg)
logger.info("Requesting Aquarium inventory...")
if not cfg.task.query:
raise ValueError("Query must be provided.")
query = {}
if cfg.task.query.user:
user = aq.User.where({"login": cfg.task.query.user})[0]
query["user_id"] = user.id
n_samples = cfg.task.query.n_samples
models = aq.Sample.last(n_samples, query)
with Progress(
"[progress.description]{task.description}",
BarColumn(),
"[progress.percentage]{task.percentage:>3.0f}%",
TimeRemainingColumn(),
TransferSpeedColumn(),
DownloadColumn(),
) as progress:
n_cpus = cfg.task.n_jobs or os.cpu_count()
task0 = progress.add_task(
"[blue]collecting Aquarium samples[/blue]", total=100
)
if cfg.task.strict:
def error_callback(e: Exception):
raise e
else:
error_callback = self.catch_constraint_error
if cfg.task.create_nodes:
task1 = progress.add_task(
"[red]writing nodes to [bold]neo4j[/bold]...[/red] ([green]cpus: {cpus}[/green])".format(
cpus=n_cpus
)
)
else:
task1 = None
with infinite_task_context(progress, task0) as callback:
node_payload = aq_samples_to_cypher(
aq,
models,
new_node_callback=callback
)
if cfg.task.create_nodes:
progress.tasks[task1].total = len(node_payload)
driver.pool(n_cpus).write(
node_payload,
callback=lambda x: progress.update(task1, advance=len(x)),
chunksize=cfg.task.chunksize,
error_callback=error_callback,
)
progress.update(task1, completed=progress.tasks[task1].total)
def upload_data(
dataset_identifier: str,
files: Optional[List[Union[PathLike, LocalFile]]],
files_to_exclude: Optional[List[PathLike]],
fps: int,
path: Optional[str],
frames: bool,
preserve_folders: bool = False,
verbose: bool = False,
) -> None:
"""
Uploads the provided files to the remote dataset.
Exits the application if no dataset with the given name is found, the files in the given path
have unsupported formats, or if there are no files found in the given Path.
Parameters
----------
dataset_identifier : str
Slug of the dataset to retrieve.
files : List[Union[PathLike, LocalFile]]
List of files to upload. Can be None.
files_to_exclude : List[PathLike]
List of files to exclude from the file scan (which is done only if files is None).
fps : int
Frame rate to split videos in.
path : Optional[str]
If provided; files will be placed under this path in the v7 platform. If `preserve_folders`
is `True` then it must be possible to draw a relative path from this folder to the one the
files are in, otherwise an error will be raised.
frames : bool
Specify whether the files will be uploaded as a list of frames or not.
preserve_folders : bool
Specify whether or not to preserve folder paths when uploading.
verbose : bool
Specify whther to have full traces print when uploading files or not.
"""
client: Client = _load_client()
try:
max_workers: int = concurrent.futures.ThreadPoolExecutor(
)._max_workers # type: ignore
dataset: RemoteDataset = client.get_remote_dataset(
dataset_identifier=dataset_identifier)
sync_metadata: Progress = Progress(
SpinnerColumn(), TextColumn("[bold blue]Syncing metadata"))
overall_progress = Progress(
TextColumn("[bold blue]{task.fields[filename]}"), BarColumn(),
"{task.completed} of {task.total}")
file_progress = Progress(
TextColumn("[bold green]{task.fields[filename]}", justify="right"),
BarColumn(),
"[progress.percentage]{task.percentage:>3.1f}%",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
)
progress_table: Table = Table.grid()
progress_table.add_row(sync_metadata)
progress_table.add_row(file_progress)
progress_table.add_row(overall_progress)
with Live(progress_table):
sync_task: TaskID = sync_metadata.add_task("")
file_tasks: Dict[str, TaskID] = {}
overall_task = overall_progress.add_task("[green]Total progress",
filename="Total progress",
total=0,
visible=False)
def progress_callback(total_file_count, file_advancement):
sync_metadata.update(sync_task, visible=False)
overall_progress.update(overall_task,
total=total_file_count,
advance=file_advancement,
visible=True)
def file_upload_callback(file_name, file_total_bytes,
file_bytes_sent):
if file_name not in file_tasks:
file_tasks[file_name] = file_progress.add_task(
f"[blue]{file_name}",
filename=file_name,
total=file_total_bytes)
# Rich has a concurrency issue, so sometimes updating progress
# or removing a task fails. Wrapping this logic around a try/catch block
# is a workaround, we should consider solving this properly (e.g.: using locks)
try:
file_progress.update(file_tasks[file_name],
completed=file_bytes_sent)
for task in file_progress.tasks:
if task.finished and len(
file_progress.tasks) >= max_workers:
file_progress.remove_task(task.id)
except Exception as e:
pass
upload_manager = dataset.push(
files,
files_to_exclude=files_to_exclude,
fps=fps,
as_frames=frames,
path=path,
preserve_folders=preserve_folders,
progress_callback=progress_callback,
file_upload_callback=file_upload_callback,
)
console = Console(theme=_console_theme())
console.print()
if not upload_manager.blocked_count and not upload_manager.error_count:
console.print(
f"All {upload_manager.total_count} files have been successfully uploaded.\n",
style="success")
return
already_existing_items = []
other_skipped_items = []
for item in upload_manager.blocked_items:
if item.reason == "ALREADY_EXISTS":
already_existing_items.append(item)
else:
other_skipped_items.append(item)
if already_existing_items:
console.print(
f"Skipped {len(already_existing_items)} files already in the dataset.\n",
style="warning",
)
if upload_manager.error_count or other_skipped_items:
error_count = upload_manager.error_count + len(other_skipped_items)
console.print(
f"{error_count} files couldn't be uploaded because an error occurred.\n",
style="error",
)
if not verbose and upload_manager.error_count:
console.print('Re-run with "--verbose" for further details')
return
error_table: Table = Table("Dataset Item ID",
"Filename",
"Remote Path",
"Stage",
"Reason",
show_header=True,
header_style="bold cyan")
for item in upload_manager.blocked_items:
if item.reason != "ALREADY_EXISTS":
error_table.add_row(str(item.dataset_item_id), item.filename,
item.path, "UPLOAD_REQUEST", item.reason)
for error in upload_manager.errors:
for local_file in upload_manager.local_files:
if local_file.local_path != error.file_path:
continue
for pending_item in upload_manager.pending_items:
if pending_item.filename != local_file.data["filename"]:
continue
error_table.add_row(
str(pending_item.dataset_item_id),
pending_item.filename,
pending_item.path,
error.stage.name,
str(error.error),
)
break
if error_table.row_count:
console.print(error_table)
print_new_version_info(client)
except NotFound as e:
_error(f"No dataset with name '{e.name}'")
except UnsupportedFileType as e:
_error(f"Unsupported file type {e.path.suffix} ({e.path.name})")
except ValueError:
_error(f"No files found")
class AbstractModelRepository(ABC):
"""
Define an abstract base class that describes a remote model repository.
Attributes
----------
name : str
The name of the remote repository.
"""
_progress = Progress(
TextColumn("{task.fields[model_id]}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
DownloadColumn(),
TransferSpeedColumn(),
TimeRemainingColumn(),
)
name: str = "Abstract"
def __init__(self, *, url: Union[httpx.URL, str], **kwargs) -> None:
"""
Initialize the model repository.
Parameters
----------
url : httpx.URL or str
The base URL from where to load the models.
Other Parameters
----------------
kwargs
Passed to the parent constructor in order to enable multiple inheritance.
"""
super().__init__(**kwargs)
self._url = httpx.URL(url=url)
@property
def url(self) -> httpx.URL:
"""Return the repository's URL."""
return self._url.copy_with()
@abstractmethod
def get_sbml(self, model_id: str) -> bytes:
"""
Attempt to download an SBML document from the repository.
Parameters
----------
model_id : str
The identifier of the desired metabolic model. This is typically repository
specific.
Returns
-------
bytes
A gzip-compressed, UTF-8 encoded SBML document.
"""
raise NotImplementedError(
"Implement `get_sbml` in a concrete child class.")
DownloadColumn,
TextColumn,
TransferSpeedColumn,
TimeRemainingColumn,
Progress,
TaskID,
)
progress = Progress(
TextColumn("[bold blue]{task.fields[filename]}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
)
def copy_url(task_id: TaskID, url: str, path: str) -> None:
"""Copy data from a url to a local file."""
response = urlopen(url)
# This will break if the response doesn't contain content length
progress.update(task_id, total=int(response.info()["Content-length"]))
with open(path, "wb") as dest_file:
progress.start_task(task_id)
for data in iter(partial(response.read, 32768), b""):
dest_file.write(data)
progress.update(task_id, advance=len(data))
class YataiClient:
log_progress = ProgressWrapper(Progress(
TextColumn("{task.description}"), ))
spinner_progress = ProgressWrapper(
Progress(
TextColumn(" "),
TimeElapsedColumn(),
TextColumn("[bold purple]{task.fields[action]}"),
SpinnerColumn("simpleDots"),
))
transmission_progress = ProgressWrapper(
Progress(
TextColumn("[bold blue]{task.description}", justify="right"),
BarColumn(bar_width=None),
"[progress.percentage]{task.percentage:>3.1f}%",
"•",
DownloadColumn(),
"•",
TransferSpeedColumn(),
"•",
TimeRemainingColumn(),
))
progress_group = Group(Panel(Group(log_progress, spinner_progress)),
transmission_progress)
@contextmanager
def spin(self, *, text: str):
task_id = self.spinner_progress.add_task("", action=text)
try:
yield
finally:
self.spinner_progress.stop_task(task_id)
self.spinner_progress.update(task_id, visible=False)
@inject
def push_bento(
self,
bento: "Bento",
*,
force: bool = False,
model_store: "ModelStore" = Provide[BentoMLContainer.model_store],
):
with Live(self.progress_group):
upload_task_id = self.transmission_progress.add_task(
f'Pushing Bento "{bento.tag}"', start=False, visible=False)
self._do_push_bento(bento,
upload_task_id,
force=force,
model_store=model_store)
@inject
def _do_push_bento(
self,
bento: "Bento",
upload_task_id: TaskID,
*,
force: bool = False,
model_store: "ModelStore" = Provide[BentoMLContainer.model_store],
):
yatai_rest_client = get_current_yatai_rest_api_client()
name = bento.tag.name
version = bento.tag.version
if version is None:
raise BentoMLException(f"Bento {bento.tag} version cannot be None")
info = bento.info
model_names = info.models
with ThreadPoolExecutor(
max_workers=max(len(model_names), 1)) as executor:
def push_model(model: "Model"):
model_upload_task_id = self.transmission_progress.add_task(
f'Pushing model "{model.tag}"', start=False, visible=False)
self._do_push_model(model, model_upload_task_id, force=force)
futures = executor.map(push_model, (model_store.get(name)
for name in model_names))
list(futures)
with self.spin(text=f'Fetching Bento repository "{name}"'):
bento_repository = yatai_rest_client.get_bento_repository(
bento_repository_name=name)
if not bento_repository:
with self.spin(
text=f'Bento repository "{name}" not found, creating now..'
):
bento_repository = yatai_rest_client.create_bento_repository(
req=CreateBentoRepositorySchema(name=name, description=""))
with self.spin(text=f'Try fetching Bento "{bento.tag}" from Yatai..'):
remote_bento = yatai_rest_client.get_bento(
bento_repository_name=name, version=version)
if (not force and remote_bento
and remote_bento.upload_status == BentoUploadStatus.SUCCESS):
self.log_progress.add_task(
f'[bold blue]Push failed: Bento "{bento.tag}" already exists in Yatai'
)
return
if not remote_bento:
labels: t.List[LabelItemSchema] = [
LabelItemSchema(key=key, value=value)
for key, value in info.labels.items()
]
apis: t.Dict[str, BentoApiSchema] = {}
models = [str(m) for m in info.models]
with self.spin(
text=f'Registering Bento "{bento.tag}" with Yatai..'):
yatai_rest_client.create_bento(
bento_repository_name=bento_repository.name,
req=CreateBentoSchema(
description="",
version=version,
build_at=info.creation_time,
manifest=BentoManifestSchema(
service=info.service,
bentoml_version=info.bentoml_version,
apis=apis,
models=models,
size_bytes=calc_dir_size(bento.path),
),
labels=labels,
),
)
with self.spin(
text=f'Getting a presigned upload url for "{bento.tag}" ..'):
remote_bento = yatai_rest_client.presign_bento_upload_url(
bento_repository_name=bento_repository.name, version=version)
with io.BytesIO() as tar_io:
bento_dir_path = bento.path
if bento_dir_path is None:
raise BentoMLException(f'Bento "{bento}" path cannot be None')
with self.spin(
text=f'Creating tar archive for Bento "{bento.tag}"..'):
with tarfile.open(fileobj=tar_io, mode="w:gz") as tar:
def filter_(
tar_info: tarfile.TarInfo,
) -> t.Optional[tarfile.TarInfo]:
if tar_info.path == "./models" or tar_info.path.startswith(
"./models/"):
return None
return tar_info
tar.add(bento_dir_path, arcname="./", filter=filter_)
tar_io.seek(0, 0)
with self.spin(text=f'Start uploading Bento "{bento.tag}"..'):
yatai_rest_client.start_upload_bento(
bento_repository_name=bento_repository.name,
version=version)
file_size = tar_io.getbuffer().nbytes
self.transmission_progress.update(upload_task_id,
completed=0,
total=file_size,
visible=True)
self.transmission_progress.start_task(upload_task_id)
def io_cb(x: int):
self.transmission_progress.update(upload_task_id, advance=x)
wrapped_file = CallbackIOWrapper(
io_cb,
tar_io,
"read",
)
finish_req = FinishUploadBentoSchema(
status=BentoUploadStatus.SUCCESS,
reason="",
)
try:
resp = requests.put(remote_bento.presigned_upload_url,
data=wrapped_file)
if resp.status_code != 200:
finish_req = FinishUploadBentoSchema(
status=BentoUploadStatus.FAILED,
reason=resp.text,
)
except Exception as e: # pylint: disable=broad-except
finish_req = FinishUploadBentoSchema(
status=BentoUploadStatus.FAILED,
reason=str(e),
)
if finish_req.status is BentoUploadStatus.FAILED:
self.log_progress.add_task(
f'[bold red]Failed to upload Bento "{bento.tag}"')
with self.spin(text="Submitting upload status to Yatai"):
yatai_rest_client.finish_upload_bento(
bento_repository_name=bento_repository.name,
version=version,
req=finish_req,
)
if finish_req.status != BentoUploadStatus.SUCCESS:
self.log_progress.add_task(
f'[bold red]Failed pushing Bento "{bento.tag}": {finish_req.reason}'
)
else:
self.log_progress.add_task(
f'[bold green]Successfully pushed Bento "{bento.tag}"')
@inject
def pull_bento(
self,
tag: t.Union[str, Tag],
*,
force: bool = False,
bento_store: "BentoStore" = Provide[BentoMLContainer.bento_store],
model_store: "ModelStore" = Provide[BentoMLContainer.model_store],
) -> "Bento":
with Live(self.progress_group):
download_task_id = self.transmission_progress.add_task(
f'Pulling bento "{tag}"', start=False, visible=False)
return self._do_pull_bento(
tag,
download_task_id,
force=force,
bento_store=bento_store,
model_store=model_store,
)
@inject
def _do_pull_bento(
self,
tag: t.Union[str, Tag],
download_task_id: TaskID,
*,
force: bool = False,
bento_store: "BentoStore" = Provide[BentoMLContainer.bento_store],
model_store: "ModelStore" = Provide[BentoMLContainer.model_store],
) -> "Bento":
try:
bento = bento_store.get(tag)
if not force:
self.log_progress.add_task(
f'[bold blue]Bento "{tag}" exists in local model store')
return bento
bento_store.delete(tag)
except NotFound:
pass
_tag = Tag.from_taglike(tag)
name = _tag.name
version = _tag.version
if version is None:
raise BentoMLException(f'Bento "{_tag}" version can not be None')
yatai_rest_client = get_current_yatai_rest_api_client()
with self.spin(text=f'Fetching bento "{_tag}"'):
remote_bento = yatai_rest_client.get_bento(
bento_repository_name=name, version=version)
if not remote_bento:
raise BentoMLException(f'Bento "{_tag}" not found on Yatai')
with ThreadPoolExecutor(max_workers=max(
len(remote_bento.manifest.models), 1)) as executor:
def pull_model(model_tag: Tag):
model_download_task_id = self.transmission_progress.add_task(
f'Pulling model "{model_tag}"', start=False, visible=False)
self._do_pull_model(
model_tag,
model_download_task_id,
force=force,
model_store=model_store,
)
futures = executor.map(pull_model, remote_bento.manifest.models)
list(futures)
with self.spin(
text=f'Getting a presigned download url for bento "{_tag}"'):
remote_bento = yatai_rest_client.presign_bento_download_url(
name, version)
url = remote_bento.presigned_download_url
response = requests.get(url, stream=True)
if response.status_code != 200:
raise BentoMLException(
f'Failed to download bento "{_tag}": {response.text}')
total_size_in_bytes = int(response.headers.get("content-length", 0))
block_size = 1024 # 1 Kibibyte
with NamedTemporaryFile() as tar_file:
self.transmission_progress.update(download_task_id,
completed=0,
total=total_size_in_bytes,
visible=True)
self.transmission_progress.start_task(download_task_id)
for data in response.iter_content(block_size):
self.transmission_progress.update(download_task_id,
advance=len(data))
tar_file.write(data)
self.log_progress.add_task(
f'[bold green]Finished downloading all bento "{_tag}" files')
tar_file.seek(0, 0)
tar = tarfile.open(fileobj=tar_file, mode="r:gz")
with fs.open_fs("temp://") as temp_fs:
for member in tar.getmembers():
f = tar.extractfile(member)
if f is None:
continue
p = Path(member.name)
if p.parent != Path("."):
temp_fs.makedirs(str(p.parent), recreate=True)
temp_fs.writebytes(member.name, f.read())
bento = Bento.from_fs(temp_fs)
for model_tag in remote_bento.manifest.models:
with self.spin(
text=f'Copying model "{model_tag}" to bento'):
copy_model(
model_tag,
src_model_store=model_store,
target_model_store=bento.
_model_store, # type: ignore
)
bento = bento.save(bento_store)
self.log_progress.add_task(
f'[bold green]Successfully pulled bento "{_tag}"')
return bento
def push_model(self, model: "Model", *, force: bool = False):
with Live(self.progress_group):
upload_task_id = self.transmission_progress.add_task(
f'Pushing model "{model.tag}"', start=False, visible=False)
self._do_push_model(model, upload_task_id, force=force)
def _do_push_model(self,
model: "Model",
upload_task_id: TaskID,
*,
force: bool = False):
yatai_rest_client = get_current_yatai_rest_api_client()
name = model.tag.name
version = model.tag.version
if version is None:
raise BentoMLException(
f'Model "{model.tag}" version cannot be None')
info = model.info
with self.spin(text=f'Fetching model repository "{name}"'):
model_repository = yatai_rest_client.get_model_repository(
model_repository_name=name)
if not model_repository:
with self.spin(
text=f'Model repository "{name}" not found, creating now..'
):
model_repository = yatai_rest_client.create_model_repository(
req=CreateModelRepositorySchema(name=name, description=""))
with self.spin(text=f'Try fetching model "{model.tag}" from Yatai..'):
remote_model = yatai_rest_client.get_model(
model_repository_name=name, version=version)
if (not force and remote_model
and remote_model.upload_status == ModelUploadStatus.SUCCESS):
self.log_progress.add_task(
f'[bold blue]Model "{model.tag}" already exists in Yatai, skipping'
)
return
if not remote_model:
labels: t.List[LabelItemSchema] = [
LabelItemSchema(key=key, value=value)
for key, value in info.labels.items()
]
with self.spin(
text=f'Registering model "{model.tag}" with Yatai..'):
yatai_rest_client.create_model(
model_repository_name=model_repository.name,
req=CreateModelSchema(
description="",
version=version,
build_at=info.creation_time,
manifest=ModelManifestSchema(
module=info.module,
metadata=info.metadata,
context=info.context,
options=info.options,
api_version=info.api_version,
bentoml_version=info.bentoml_version,
size_bytes=calc_dir_size(model.path),
),
labels=labels,
),
)
with self.spin(
text=f'Getting a presigned upload url for model "{model.tag}"..'
):
remote_model = yatai_rest_client.presign_model_upload_url(
model_repository_name=model_repository.name, version=version)
with io.BytesIO() as tar_io:
bento_dir_path = model.path
with self.spin(
text=f'Creating tar archive for model "{model.tag}"..'):
with tarfile.open(fileobj=tar_io, mode="w:gz") as tar:
tar.add(bento_dir_path, arcname="./")
tar_io.seek(0, 0)
with self.spin(text=f'Start uploading model "{model.tag}"..'):
yatai_rest_client.start_upload_model(
model_repository_name=model_repository.name,
version=version)
file_size = tar_io.getbuffer().nbytes
self.transmission_progress.update(
upload_task_id,
description=f'Uploading model "{model.tag}"',
total=file_size,
visible=True,
)
self.transmission_progress.start_task(upload_task_id)
def io_cb(x: int):
self.transmission_progress.update(upload_task_id, advance=x)
wrapped_file = CallbackIOWrapper(
io_cb,
tar_io,
"read",
)
finish_req = FinishUploadModelSchema(
status=ModelUploadStatus.SUCCESS,
reason="",
)
try:
resp = requests.put(remote_model.presigned_upload_url,
data=wrapped_file)
if resp.status_code != 200:
finish_req = FinishUploadModelSchema(
status=ModelUploadStatus.FAILED,
reason=resp.text,
)
except Exception as e: # pylint: disable=broad-except
finish_req = FinishUploadModelSchema(
status=ModelUploadStatus.FAILED,
reason=str(e),
)
if finish_req.status is ModelUploadStatus.FAILED:
self.log_progress.add_task(
f'[bold red]Failed to upload model "{model.tag}"')
with self.spin(text="Submitting upload status to Yatai"):
yatai_rest_client.finish_upload_model(
model_repository_name=model_repository.name,
version=version,
req=finish_req,
)
if finish_req.status != ModelUploadStatus.SUCCESS:
self.log_progress.add_task(
f'[bold red]Failed pushing model "{model.tag}" : {finish_req.reason}'
)
else:
self.log_progress.add_task(
f'[bold green]Successfully pushed model "{model.tag}"')
@inject
def pull_model(
self,
tag: t.Union[str, Tag],
*,
force: bool = False,
model_store: "ModelStore" = Provide[BentoMLContainer.model_store],
) -> "Model":
with Live(self.progress_group):
download_task_id = self.transmission_progress.add_task(
f'Pulling model "{tag}"', start=False, visible=False)
return self._do_pull_model(tag,
download_task_id,
force=force,
model_store=model_store)
@inject
def _do_pull_model(
self,
tag: t.Union[str, Tag],
download_task_id: TaskID,
*,
force: bool = False,
model_store: "ModelStore" = Provide[BentoMLContainer.model_store],
) -> "Model":
try:
model = model_store.get(tag)
if not force:
self.log_progress.add_task(
f'[bold blue]Model "{tag}" already exists locally, skipping'
)
return model
else:
model_store.delete(tag)
except NotFound:
pass
yatai_rest_client = get_current_yatai_rest_api_client()
_tag = Tag.from_taglike(tag)
name = _tag.name
version = _tag.version
if version is None:
raise BentoMLException(f'Model "{_tag}" version cannot be None')
with self.spin(
text=f'Getting a presigned download url for model "{_tag}"..'):
remote_model = yatai_rest_client.presign_model_download_url(
name, version)
if not remote_model:
raise BentoMLException(f'Model "{_tag}" not found on Yatai')
url = remote_model.presigned_download_url
response = requests.get(url, stream=True)
if response.status_code != 200:
raise BentoMLException(
f'Failed to download model "{_tag}": {response.text}')
total_size_in_bytes = int(response.headers.get("content-length", 0))
block_size = 1024 # 1 Kibibyte
with NamedTemporaryFile() as tar_file:
self.transmission_progress.update(
download_task_id,
description=f'Downloading model "{_tag}"',
total=total_size_in_bytes,
visible=True,
)
self.transmission_progress.start_task(download_task_id)
for data in response.iter_content(block_size):
self.transmission_progress.update(download_task_id,
advance=len(data))
tar_file.write(data)
self.log_progress.add_task(
f'[bold green]Finished downloading model "{_tag}" files')
tar_file.seek(0, 0)
tar = tarfile.open(fileobj=tar_file, mode="r:gz")
with fs.open_fs("temp://") as temp_fs:
for member in tar.getmembers():
f = tar.extractfile(member)
if f is None:
continue
p = Path(member.name)
if p.parent != Path("."):
temp_fs.makedirs(str(p.parent), recreate=True)
temp_fs.writebytes(member.name, f.read())
model = Model.from_fs(temp_fs).save(model_store)
self.log_progress.add_task(
f'[bold green]Successfully pulled model "{_tag}"')
return model
