def _set_body(self, data):
"""Set body."""
if self.compressed == 'gzip':
self.body += gzip_decompress(data)
elif self.compressed == 'deflate':
self.body += zlib_decompress(data)
else:
self.body += data
def decrypt(data: bytes, password: Union[str, bytes]) -> bytes:
"""
decrypt data
:param data: encrypted data
:param password: password
:return: plain data
"""
__data = gzip_decompress(data[4:]) if data.startswith(b'moca') else data
iv, cipher = __data[:AES.block_size], __data[AES.block_size:]
return __create_aes(password, iv).decrypt(cipher)
def open(request):
request = request_vim_to_python(request)
rhandler = HTTPRedirectHandler()
rhandler.max_redirections = request['max_redirect']
opener = build_opener(rhandler)
if request['username']:
passmgr = HTTPPasswordMgrWithDefaultRealm()
passmgr.add_password(
None,
request['url'],
request['username'],
request['password'],
)
opener.add_handler(HTTPBasicAuthHandler(passmgr))
opener.add_handler(HTTPDigestAuthHandler(passmgr))
req = Request(
url=request['url'],
data=request['data'],
headers=request['headers'],
method=request['method'],
)
if request['gzip_decompress']:
req.add_header('Accept-encoding', 'gzip')
try:
res = retry(tries=request['retry'])(opener.open)(
req, timeout=request['timeout'])
except HTTPError as e:
res = e
if not hasattr(res, 'version'):
# urllib2 does not have 'version' field
import httplib
res.version = httplib.HTTPConnection._http_vsn
response_status = "HTTP/%s %d %s\n" % (
'1.1' if res.version == 11 else '1.0',
res.code,
res.msg,
)
response_headers = str(res.headers)
response_body = res.read()
if (request['gzip_decompress']
and res.headers.get('Content-Encoding') == 'gzip'):
response_body = gzip_decompress(response_body)
if hasattr(res.headers, 'get_content_charset'):
# Python 3
response_encoding = res.headers.get_content_charset()
else:
# Python 2
response_encoding = res.headers.getparam('charset')
response_body = response_body.decode(response_encoding)
return (
request['url'],
response_status + response_headers,
response_body,
)
def download_file(self):
""" """
""" If we haven't already downloaded url """
if self.content:
return self.content
""" Retrieve url and content """
auth = None
if self.auth_type:
auth_type = AUTH_TYPE_CLASSES.get(self.auth_type)
if auth_type:
auth = auth_type(self.user, self.password)
logger.debug("Try to get URL {}".format(self.url))
try:
response = requests.request(
self.method,
self.url,
data=self.post_data if self.method == "POST" else None,
headers=self.custom_headers,
auth=auth,
allow_redirects=True,
proxies=get_proxy(),
timeout=(2.0, 2.0))
# logger.info("URL '{}' retrieved, status code = {}".format(self.url, response.status_code))
assert response.status_code == 200, "Response code is not 200 ({})".format(
response.status_code)
""" If its a .gz file, dezip-it """
if self.url[-3:] == ".gz":
self.filename = self.url.split('/')[-1][:-3]
return gzip_decompress(response.content)
if response.headers.get("Content-Disposition"):
match = REGEX_GZ.search(
response.headers.get("Content-Disposition"))
if match and match[1][-3:] == ".gz":
self.filename = match[1][:-3]
return gzip_decompress(response.content)
if not self.filename:
self.filename = self.url.split('/')[-1]
except Exception as e:
raise VultureSystemError(str(e), "download '{}'".format(self.url))
return response.content
