def _(event):
"""After download, execute command."""
if event.target is not self.download_thread:
return
cmd = event.data.module.config.get("runafterdownload")
default_cmd = setting.get("runafterdownload")
commands = []
if cmd:
commands.append(cmd)
if default_cmd and default_cmd not in commands:
commands.append(default_cmd)
for command in commands:
command += " " + quote(path_join(
profile(event.data.module.config["savepath"]),
safefilepath(event.data.title)
))
try:
await_(subprocess.call, command, shell=True) # nosec
except (OSError, subprocess.SubprocessError):
traceback.print_exc()
def run_command():
for command in commands:
target = quote(
path_join(
profile(event.data.module.config["savepath"]),
safefilepath(event.data.title)))
if "{target}" in command:
command = command.format(target=target)
else:
command += " " + target
print(f"run command: {command}")
try:
await_(subprocess.call, command, shell=True) # nosec
except (OSError, subprocess.SubprocessError):
traceback.print_exc()
def grabber(url,
header=None,
*,
referer=None,
cookie=None,
raise_429=True,
params=None,
done=None):
"""Request url, return text or bytes of the content."""
_scheme, netloc, _path, _query, _frag = urlsplit(url)
if netloc not in sessions:
s = requests.Session()
s.headers.update(default_header)
sessions[netloc] = s
else:
s = sessions[netloc]
if header:
s.headers.update(header)
if referer:
s.headers['referer'] = quote_unicode(referer)
if cookie:
quote_unicode_dict(cookie)
requests.utils.add_dict_to_cookiejar(s.cookies, cookie)
r = await_(do_request, s, url, params, raise_429)
if done:
done(s, r)
return r
def grabber(url, header=None, *, referer=None, cookie=None,
retry=False, params=None, done=None, proxy=None,
method="GET", data=None):
"""Request url, return text or bytes of the content."""
_scheme, netloc, _path, _query, _frag = urlsplit(url)
if netloc not in sessions:
s = requests.Session()
s.headers.update(default_header)
sessions[netloc] = s
else:
s = sessions[netloc]
if header:
s.headers.update(header)
if referer:
s.headers['referer'] = quote_unicode(referer)
if cookie:
quote_unicode_dict(cookie)
requests.utils.add_dict_to_cookiejar(s.cookies, cookie)
if isinstance(proxy, str):
proxies = {'http': proxy, 'https': proxy}
else:
proxies = proxy
r = await_(do_request, s, url, params, proxies, method, data, retry)
if done:
done(s, r)
return r
def grabber(url,
header=None,
*,
referer=None,
cookie=None,
retry=False,
done=None,
proxy=None,
**kwargs):
"""Request url, return text or bytes of the content."""
_scheme, netloc, _path, _query, _frag = urlsplit(url)
s = get_session(netloc)
if header:
s.headers.update(header)
if referer:
s.headers['referer'] = quote_unicode(referer)
if cookie:
quote_unicode_dict(cookie)
requests.utils.add_dict_to_cookiejar(s.cookies, cookie)
if isinstance(proxy, str):
proxies = {'http': proxy, 'https': proxy}
else:
proxies = proxy
r = await_(do_request, s, url, proxies, retry, **kwargs)
if done:
done(s, r)
return r
def test_await(self):
from worker import await_, later
from time import sleep
a = False
def blocking_task():
sleep(1)
def task():
nonlocal a
a = True
later(task)
# ensure await_ enter the event loop
await_(blocking_task)
self.assertTrue(a)
def _(event):
"""After download, execute command."""
if event.target is not self.download_thread:
return
cmd = event.data.module.config.get("runafterdownload")
default_cmd = setting.get("runafterdownload")
commands = []
if cmd:
commands.append(cmd)
if default_cmd and default_cmd not in commands:
commands.append(default_cmd)
for command in commands:
command += " " + quote(
path_join(profile(event.data.module.config["savepath"]),
safefilepath(event.data.title)))
try:
await_(subprocess.call, command, shell=True) # nosec
except (OSError, subprocess.SubprocessError):
traceback.print_exc()
def grabber(url,
header=None,
*,
referer=None,
cookie=None,
retry=False,
params=None,
done=None,
proxy=None,
method="GET",
data=None):
"""Request url, return text or bytes of the content."""
_scheme, netloc, _path, _query, _frag = urlsplit(url)
if netloc not in sessions:
s = requests.Session()
s.headers.update(default_header)
sessions[netloc] = s
else:
s = sessions[netloc]
if header:
s.headers.update(header)
if referer:
s.headers['referer'] = quote_unicode(referer)
if cookie:
quote_unicode_dict(cookie)
requests.utils.add_dict_to_cookiejar(s.cookies, cookie)
if isinstance(proxy, str):
proxies = {'http': proxy, 'https': proxy}
else:
proxies = proxy
r = await_(do_request, s, url, params, proxies, method, data, retry)
if done:
done(s, r)
return r