async def download(**kwargs):
self = dbg.__self__
temp = mktemp()
rq = DlRequest(file_path=temp.pathname,
max_retries=dbg.config['max_retries'])
for uri in self.dumps():
uri.pop('id')
rq.put(**uri)
exception = None
async with dlopen(rq) as dl:
dbg.upload(
size=dl.file.size,
pathname=dl.file.pathname,
)
dbg.set_percent(dl.percent_complete)
dbg.set_timeleft(dl.remaining_time)
dbg.set_speed(lambda: dl.transfer_rate())
dl.start(loop=asyncio.get_running_loop())
while not dl._future:
await asyncio.sleep(0.01)
# 添加请求停止器
dbg.add_stopper(dl.pause)
async for exception in dl.aexceptions():
dbg.warning(exception.exc_info)
if isinstance(exception, HandlerError):
await dl.apause()
break
else:
exception = None
await dl.ajoin()
if exception:
# 若发生异常，抛出异常
raise exception from exception.exception
# 更新文件信息
dbg.upload(
pathname=dl.file.pathname,
size=dl.file.size,
)
async def end_request(self):
temp = dbg.tempdir.mktemp()
req = DlRequest(file_path=temp.filepath)
for uri in self.dumps():
uri.pop('id')
req.put(**uri)
async with dlopen(req) as dl:
dbg.upload(
size=dl.file.size,
filepath=dl.file.pathname,
)
dbg.set_percent(dl.percent_complete)
dbg.set_timeleft(dl.remaining_time)
dbg.set_speed(lambda: dl.transfer_rate())
dl.start(loop=asyncio.get_running_loop())
# FIX: Nbdler 下载器在协程下出现的问题
while not dl._future:
await asyncio.sleep(0.01)
dbg.add_stopper(dl.pause)
async for exception in dl.aexceptions():
dbg.warning(exception.exc_info)
if isinstance(exception, HandlerError):
await dl.apause()
break
else:
exception = None
await dl.ajoin()
if exception:
# 若发生异常，抛出异常
raise exception from exception.exception
# 更新文件信息
dbg.upload(
filepath=dl.file.pathname,
size=dl.file.size,
)
def handle_font_remap(bs4):
font_style_text = None
for i in bs4.find_all('style'):
if 'font-face' in i.string:
font_style_text = i.string
break
if font_style_text is None:
raise ValueError('cannot find font-face.')
rex_url = re.compile('url\("(.*?)"\)', re.I)
font_url = rex_url.search(font_style_text).group(1)
name = './fonts/%s.woff' % time.perf_counter_ns()
req = Request(url=font_url, headers=simple_spider.HEADERS, filepath=name)
dl = dlopen(req)
dl.start()
dl.trap()
dl.close()
remap_key = parse_font(name)
return remap_key
import nbdler
import time
req = nbdler.Request(
'http://mirrors.neusoft.edu.cn/centos/7.7.1908/isos/x86_64/CentOS-7-x86_64-Minimal-1908.iso',
max_thread=200,
block_size=1024)
req.put(
'http://mirrors.zju.edu.cn/centos/7.7.1908/isos/x86_64/CentOS-7-x86_64-Minimal-1908.iso'
)
req.put(
'http://ftp.sjtu.edu.cn/centos/7.7.1908/isos/x86_64/CentOS-7-x86_64-Minimal-1908.iso'
)
dl = nbdler.dlopen(req)
dl.start()
dl.pause()
dl.start()
dl.pause()
del dl
dl = nbdler.dlopen('CentOS-7-x86_64-Minimal-1908.iso.nb')
dl.start()
while not dl.is_finished():
print('inst: %f KB/S, remain_time: %f s, remain_byte: %d, online: %d' %
(dl.getinstspeed() / 1024, dl.get_time_left(), dl.get_byte_left(),
dl.get_online_cnt()))
time.sleep(0.5)
dl.close()
async def download(uri: str = None,
headers: Dict = None,
*,
multi_sources: List[Dict] = None,
**kwargs):
""" 下载请求
Args:
uri: 下载uri
headers: 指定下载请求头
multi_sources: 多下载源的添加方式。
[{'uri': 'http://xxx', 'headers': headers}, ...]
"""
def speed():
nonlocal dl
transfer_rate = dl.transfer_rate()
return f'{readable_file_size(transfer_rate)}/s'
# unitdict = {
# 'GB/s': 1024 * 1024 * 1024,
# 'MB/s': 1024 * 1024,
# 'KB/s': 1024,
# 'B/s': 1,
# }
# for k, v in unitdict.items():
# if transfer_rate > v:
# return f'{round(transfer_rate / v, 2)} {k}'
# return f'{round(transfer_rate / v, 2)} B/s'
# 创建下载请求对象
tempf = ctx.tempdir.mktemp()
dlr = DlRequest(file_path=tempf.filepath)
sources = []
if uri:
sources = [{'uri': uri, 'headers': headers, **kwargs}]
sources += multi_sources or []
for source in sources:
dlr.put(**source)
async with dlopen(dlr) as dl:
ctx.upload(
filesize=dl.file.size,
dstpath=dl.file.pathname,
downloadSize=lambda: readable_file_size(dl.walk_length()),
writeSize=lambda: readable_file_size(dl.done_length()),
)
ctx.set_percent(dl.percent_complete)
ctx.set_timeleft(dl.remaining_time)
ctx.set_speed(speed)
dl.start(loop=asyncio.get_running_loop())
# FIX: Nbdler 下载器在协程下出现的问题
while not dl._future:
await asyncio.sleep(0.01)
# 创建下载停止器
ctx.add_stopper(dl.pause)
async for exception in dl.aexceptions():
ctx.warning(exception.exc_info)
if isinstance(exception, HandlerError):
await dl.apause()
break
else:
exception = None
await dl.ajoin()
if exception:
# 若发生异常，抛出异常
raise exception from exception.exception
# 更新文件信息
ctx.upload(
dstpath=dl.file.pathname,
filesize=dl.file.size,
downloadSize=lambda: readable_file_size(dl.walk_length()),
writeSize=lambda: readable_file_size(dl.done_length()),
)