def handle_401(self, r: R, **kwargs):
"""Takes the given response and re-tries auth with a new nonce."""
if r.status_code == 401 and self.num_401_calls < 2:
self.num_401_calls += 1
# Renew nonce
nonce = self.new_nonce(max(self._nonce(), self.last_nonce + 1))
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.close()
prep = r.request.copy()
cookies = prep._cookies
requests.auth.extract_cookies_to_jar(cookies, r.request, r.raw)
prep.prepare_cookies(cookies)
self.auth_request(prep, nonce)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
return r
def stops_incrementing(
counter: Value, response: requests.Response, tolerance: int = 15
) -> bool:
# Check that a counter stops incrementing after the response is closed.
# tolerance (int):
# Maximum number of events the server is allowed to send after the
# connection has been closed.
def wait_for_events(expect_many=False):
nonlocal counter
num_before = counter.value
time.sleep(0.1)
num_after = counter.value
if expect_many:
assert num_after - num_before >= 5 * tolerance, (
num_after,
num_before,
tolerance,
)
return num_after
sent_before_closing = wait_for_events(expect_many=True)
response.close()
sent_after_closing = wait_for_events() - sent_before_closing
assert sent_after_closing <= tolerance, (sent_after_closing, tolerance)
return True
def handle_401(self, r: R, **kwargs):
"""Takes the given response and re-tries auth with a new nonce."""
if r.status_code == 401 and self.num_401_calls < 2:
self.num_401_calls += 1
# Renew nonce
nonce = self.new_nonce(max(self._nonce(), self.last_nonce + 1))
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.close()
prep = r.request.copy()
cookies = prep._cookies
requests.auth.extract_cookies_to_jar(cookies, r.request, r.raw)
prep.prepare_cookies(cookies)
self.auth_request(prep, nonce)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
return r
def _decode_response(
self, response: requests.Response
) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
"""Decode the response as JSON entry or return a dictionary with the error
Parameters
----------
response: requests.Response
Response to decode
debug : bool
Debugging flag. In this case failed requests get printed
Returns
-------
dict
JSON decoded entry or error
"""
if "latest-darwin-py" in response.headers:
self._handle_latest_darwin_py(response.headers["latest-darwin-py"])
try:
return response.json()
except ValueError:
self.log.error(f"[ERROR {response.status_code}] {response.text}")
response.close()
return {
"error": "Response is not JSON encoded",
"status_code": response.status_code,
"text": response.text
}
def _download(self,
response: Response,
fobj: Optional[BytesIO] = None,
chunk_size: int = 1024,
stream_hook: Optional[Callable[[Response,
BytesIO,
int
], BytesIO
]] = None,
hook_kwargs: Optional[Dict] = None
) -> BytesIO:
'''
File download manager for Nessus
Args:
response (Response): The Response object to work on
fobj (BytesIO, optional):
The file object to write to. If unspecified, an in-memory
object will be created and returned
chunk_size (int, optional):
The chunk size to use when writing to the file object. The
default is unspecified is ``1024`` bytes.
stream_hook (Callable[Response, BytesIO, int], optional):
If specified, the output will be passed to the stream hook
instead of processing ourselves.
hook_kwargs (Dict, optional):
Any additional keyword arguments that should be passed on to
the stream hook.
Returns:
BytesIO:
The file object
'''
def base_hook(resp: Response,
fobj: BytesIO,
chunk_size: int,
**kwargs
):
''' Default stream hook '''
for chunk in resp.iter_content(chunk_size=chunk_size):
if chunk:
fobj.write(chunk)
# Set the default attributes values if nothing was passed to them
if fobj is None:
fobj = BytesIO()
if hook_kwargs is None:
hook_kwargs = {}
if stream_hook is None:
stream_hook = base_hook
# Call the stream hook with the Response object passed to us
stream_hook(response, fobj, chunk_size, **hook_kwargs)
# seek the file back to the beginning, close the response, and return
# the file object to the caller
fobj.seek(0)
response.close()
return fobj
def _process_content(response: requests.Response, max_size: int) -> str:
"""
Checks content for allowed size and return it
:param response:
:param max_size:
:return: content string
"""
content = ''
for chunk in response.iter_content(1024, decode_unicode=True):
content += chunk
if len(content) > max_size:
response.close()
raise SizeException(f'File size is too large, {response.url}')
return content
def post(url, data, headers, verify):
self.assertEqual(
url,
'http://localhost:8080/auth/realms/teamplay/protocol/openid-connect/token'
)
self.assertTrue(
ordered(data) == ordered(
{
'code': 'mycode',
'scope': 'abc def',
'grant_type': 'authorization_code',
'client_id': 'thetick',
'redirect_uri': 'http://yourwebsite.com/redirect'
}))
self.assertTrue(
ordered(headers) == ordered(
{
'Content-Type':
'application/x-www-form-urlencoded;charset=UTF-8',
'Accept': 'application/json',
'Authorization': 'Basic dGhldGljazphcnRodXI='
}))
response = Response()
response.status_code = 200
response.close = close
response.json = lambda: {'access_token': 'I grant thy access'}
return response
def send(self,
request,
stream=None,
timeout=None,
verify=None,
cert=None,
proxies=None):
pathname = url_to_path(request.url)
resp = Response()
resp.status_code = 200
resp.url = request.url
try:
stats = stat(pathname)
except OSError as exc:
resp.status_code = 404
resp.raw = exc
else:
modified = formatdate(stats.st_mtime, usegmt=True)
content_type = guess_type(pathname)[0] or "text/plain"
resp.headers = CaseInsensitiveDict({
"Content-Type": content_type,
"Content-Length": stats.st_size,
"Last-Modified": modified,
})
resp.raw = open(pathname, "rb")
resp.close = resp.raw.close
return resp
def send(self, request, stream=None, timeout=None, verify=None, cert=None, proxies=None):
pathname = url_to_path(request.url)
resp = Response()
resp.status_code = 200
resp.url = request.url
try:
stats = stat(pathname)
except OSError as exc:
resp.status_code = 404
resp.raw = exc
else:
modified = formatdate(stats.st_mtime, usegmt=True)
content_type = guess_type(pathname)[0] or "text/plain"
resp.headers = CaseInsensitiveDict({
"Content-Type": content_type,
"Content-Length": stats.st_size,
"Last-Modified": modified,
})
resp.raw = open(pathname, "rb")
resp.close = resp.raw.close
return resp
def handle_401(self, r: Response, **kwargs) -> Response:
if r.status_code != 401:
return r
if r.headers['Content-Type'].lower() == 'application/json':
res_data = r.json()
try:
message = res_data['error']['message']
except KeyError:
return r
if message != 'token invalid':
return r
if self._thread_local.pos is not None:
# Rewind the file position indicator of the body to where
# it was to resend the request.
# noinspection PyUnresolvedReferences
r.request.body.seek(self._thread_local.pos)
if self._thread_local.num_401_calls < 2:
self._thread_local.num_401_calls += 1
# Consume content and release the original connection
# to allow our new request to reuse the same one.
# noinspection PyStatementEffect
r.content
r.close()
prep = r.request.copy()
# noinspection PyUnresolvedReferences,PyProtectedMember
extract_cookies_to_jar(prep._cookies, r.request, r.raw)
# noinspection PyUnresolvedReferences,PyProtectedMember
prep.prepare_cookies(prep._cookies)
self.invalidate_token()
self.apply_authorization_header(prep)
# noinspection PyUnresolvedReferences
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
self._thread_local.num_401_calls = 1
return r
def send(self,
request,
stream=None,
timeout=None,
verify=None,
cert=None,
proxies=None):
resp = Response()
resp.status_code = 200
resp.url = request.url
try:
import boto
except ImportError:
stderrlog.info('\nError: boto is required for S3 channels. '
'Please install it with `conda install boto`\n'
'Make sure to run `source deactivate` if you '
'are in a conda environment.\n')
resp.status_code = 404
return resp
conn = boto.connect_s3()
bucket_name, key_string = url_to_s3_info(request.url)
# Get the bucket without validation that it exists and that we have
# permissions to list its contents.
bucket = conn.get_bucket(bucket_name, validate=False)
try:
key = bucket.get_key(key_string)
except boto.exception.S3ResponseError as exc:
# This exception will occur if the bucket does not exist or if the
# user does not have permission to list its contents.
resp.status_code = 404
resp.raw = exc
return resp
if key and key.exists:
modified = key.last_modified
content_type = key.content_type or "text/plain"
resp.headers = CaseInsensitiveDict({
"Content-Type": content_type,
"Content-Length": key.size,
"Last-Modified": modified,
})
_, self._temp_file = mkstemp()
key.get_contents_to_filename(self._temp_file)
f = open(self._temp_file, 'rb')
resp.raw = f
resp.close = resp.raw.close
else:
resp.status_code = 404
return resp
def get(url, verify):
response = Response()
response.status_code = 200
response.close = close
if url == 'http://localhost:8080/auth/realms/teamplay/.well-known/openid-configuration':
response.json = lambda: json.loads(WELL_KNOWN)
elif url == 'http://localhost:8080/auth/realms/teamplay/protocol/openid-connect/certs':
response.json = lambda: json.loads(KEYS)
else:
self.fail('Unexpected url: {}'.format(url))
return response
def send(self, request, stream=None, timeout=None, verify=None, cert=None, proxies=None):
resp = Response()
resp.status_code = 200
resp.url = request.url
try:
import boto
except ImportError:
stderrlog.info('\nError: boto is required for S3 channels. '
'Please install it with `conda install boto`\n'
'Make sure to run `source deactivate` if you '
'are in a conda environment.\n')
resp.status_code = 404
return resp
conn = boto.connect_s3()
bucket_name, key_string = url_to_s3_info(request.url)
# Get the bucket without validation that it exists and that we have
# permissions to list its contents.
bucket = conn.get_bucket(bucket_name, validate=False)
try:
key = bucket.get_key(key_string)
except boto.exception.S3ResponseError as exc:
# This exception will occur if the bucket does not exist or if the
# user does not have permission to list its contents.
resp.status_code = 404
resp.raw = exc
return resp
if key and key.exists:
modified = key.last_modified
content_type = key.content_type or "text/plain"
resp.headers = CaseInsensitiveDict({
"Content-Type": content_type,
"Content-Length": key.size,
"Last-Modified": modified,
})
_, self._temp_file = mkstemp()
key.get_contents_to_filename(self._temp_file)
f = open(self._temp_file, 'rb')
resp.raw = f
resp.close = resp.raw.close
else:
resp.status_code = 404
return resp
def __init__(self,
requests_response: requests.Response,
max_size: Optional[int],
error_is_client_side: bool = False):
"""Constructor."""
try:
# Read the raw data right away without waiting for a call to raw_data() to make sure that the server doesn't
# time out while returning stuff
self.__response_data = self.__read_response_data(
requests_response=requests_response, max_size=max_size)
# Release the response to return connection back to the pool
# (http://docs.python-requests.org/en/master/user/advanced/#body-content-workflow)
requests_response.close()
except Exception as ex:
log.warning("Error reading data for URL {}: {}".format(
requests_response.url, str(ex)))
error_response = requests.Response()
error_response.status_code = HTTPStatus.REQUEST_TIMEOUT.value
error_response.reason = HTTPStatus.REQUEST_TIMEOUT.phrase
error_response.request = requests_response.request
error_response.history = []
requests_response = error_response
# We treat timeouts as client-side errors too because we can retry on them
error_is_client_side = True
self.__response_data = str(ex).encode('utf-8')
self.__requests_response = requests_response
self.__error_is_client_side = error_is_client_side
self.__previous_response = None
self.__request = None
def handle_saf(res: requests.Response, ignore_limited: bool = False):
try:
for line in res.iter_lines(decode_unicode=True):
if not line:
continue
try:
saf_msg = json.loads(line)
except json.JSONDecodeError as e:
raise dnsdb2.ProtocolError(
f'could not decode json: {line}') from e
cond = saf_msg.get('cond')
obj = saf_msg.get('obj')
msg = saf_msg.get('msg')
if cond == COND_BEGIN:
continue
elif cond == COND_SUCCEEDED:
return
if obj:
yield obj
if cond == COND_ONGOING or not cond:
continue
elif cond == COND_LIMITED:
if ignore_limited:
return
raise dnsdb2.QueryLimited(msg)
elif cond == COND_FAILED:
raise dnsdb2.QueryFailed(msg)
else:
raise dnsdb2.ProtocolError(f'invalid cond: {cond}')
raise dnsdb2.QueryTruncated()
finally:
res.close()
def __init__(self,
requests_response: requests.Response,
max_size: Optional[int],
error_is_client_side: bool = False):
"""Constructor."""
try:
# Read the raw data right away without waiting for a call to raw_data() to make sure that the server doesn't
# time out while returning stuff
self.__response_data = self.__read_response_data(requests_response=requests_response, max_size=max_size)
# Release the response to return connection back to the pool
# (http://docs.python-requests.org/en/master/user/advanced/#body-content-workflow)
requests_response.close()
except Exception as ex:
log.warning("Error reading data for URL {}: {}".format(requests_response.url, str(ex)))
error_response = requests.Response()
error_response.status_code = HTTPStatus.REQUEST_TIMEOUT.value
error_response.reason = HTTPStatus.REQUEST_TIMEOUT.phrase
error_response.request = requests_response.request
error_response.history = []
requests_response = error_response
# We treat timeouts as client-side errors too because we can retry on them
error_is_client_side = True
self.__response_data = str(ex).encode('utf-8')
self.__requests_response = requests_response
self.__error_is_client_side = error_is_client_side
self.__previous_response = None
self.__request = None
def send(self, request, stream=False, timeout=None, **kwargs):
parsed_url = urllib.parse.urlparse(request.url)
file_path = parsed_url.path
# Strip the leading slash, if present.
if file_path.startswith('/'):
file_path = file_path[1:]
try:
self.conn = self.get_connection(parsed_url.netloc, timeout)
except ConnectionRefusedError as exc:
# Wrap this in a requests exception.
# in requests 2.2.1, ConnectionError does not take keyword args
raise requests.exceptions.ConnectionError() from exc
resp = Response()
resp.url = request.url
try:
size = self.conn.size(file_path)
except error_perm:
resp.status_code = 404
return resp
if stream:
# We have to do this in a background thread, since ftplib's and requests' approaches are the opposite:
# ftplib is callback based, and requests needs to expose an iterable. (Push vs pull)
# When the queue size is reached, puts will block. This provides some backpressure.
queue = Queue(maxsize=100)
done_sentinel = object()
def handle_transfer():
# Download all the chunks into a queue, then place a sentinel object into it to signal completion.
self.conn.retrbinary('RETR ' + file_path, queue.put)
queue.put(done_sentinel)
Thread(target=handle_transfer).start()
def stream(amt=8192, decode_content=False):
"""A generator, yielding chunks from the queue."""
# We maintain a buffer so the consumer gets exactly the number of bytes requested.
buffer = bytearray()
while True:
data = queue.get()
if data is not done_sentinel:
buffer.extend(data)
if len(buffer) >= amt:
result = buffer[:amt]
buffer = buffer[amt:]
yield result
else:
if buffer:
yield buffer
return
Raw = namedtuple('raw', 'stream')
raw = Raw(stream)
resp.status_code = 200
resp.raw = raw
resp.headers['content-length'] = size
resp.close = lambda: self.conn.close()
return resp
else:
# Not relevant for Ubuntu Make.
raise NotImplementedError
def __read_response_data(self,
requests_response: requests.Response) -> str:
"""Read data from Response object. Raises on read errors, callers are expected to catch exceptions."""
max_size = self.max_size()
response_data = ""
read_response_data = True
if max_size is not None:
content_length = requests_response.headers.get(
'Content-Length', None)
try:
if content_length is not None:
# HTTP spec allows one to combine multiple headers into one so Content-Length might look
# like "Content-Length: 123, 456"
if ',' in content_length:
content_length = content_length.split(',')
content_length = list(map(int, content_length))
content_length = max(content_length)
content_length = int(content_length)
except Exception as ex:
log.warning(
"Unable to read Content-Length for URL '%(url)s': %(exception)s"
% {
'url': requests_response.url,
'exception': str(ex),
})
content_length = None
if content_length is not None:
if content_length > max_size:
log.warning("Content-Length exceeds %d for URL %s" % (
max_size,
requests_response.url,
))
# Release the response to return connection back to the pool
# (http://docs.python-requests.org/en/master/user/advanced/#body-content-workflow)
requests_response.close()
read_response_data = False
if read_response_data:
# requests's "apparent_encoding" is not used because chardet might OOM on big binary data responses
encoding = requests_response.encoding
if encoding is not None:
# If "Content-Type" HTTP header contains a string "text" and doesn't have "charset" property,
# "requests" falls back to setting the encoding to ISO-8859-1, which is probably not right
# (encoding might have been defined in the HTML content itself via <meta> tag), so we use the
# "apparent encoding" instead
if encoding.lower() == 'iso-8859-1':
# Will try to auto-detect later
encoding = None
# Some pages report some funky encoding; in that case, fallback to UTF-8
if encoding is not None:
try:
codecs.lookup(encoding)
except LookupError:
log.warning("Invalid encoding %s for URL %s" % (
encoding,
requests_response.url,
))
# Autodetect later
encoding = None
# 100 KB should be enough for for chardet to be able to make an informed decision
chunk_size = 1024 * 100
decoder = None
response_data_size = 0
for chunk in requests_response.raw.stream(chunk_size,
decode_content=True):
if encoding is None:
# Test the encoding guesser's opinion, just like browsers do
try:
encoding = chardet.detect(chunk)['encoding']
except Exception as ex:
log.warning(
"Unable to detect encoding for URL %s: %s" % (
requests_response.url,
str(ex),
))
encoding = None
# If encoding is not in HTTP headers nor can be determined from content itself, assume that
# it's UTF-8
if encoding is None:
encoding = 'UTF-8'
if decoder is None:
decoder = codecs.getincrementaldecoder(encoding)(
errors='replace')
decoded_chunk = decoder.decode(chunk)
response_data += decoded_chunk
response_data_size += len(
chunk) # byte length, not string length
# Content-Length might be missing / lying, so we measure size while fetching the data too
if max_size is not None:
if response_data_size > max_size:
log.warning("Data size exceeds %d for URL %s" % (
max_size,
requests_response.url,
))
# Release the response to return connection back to the pool
# (http://docs.python-requests.org/en/master/user/advanced/#body-content-workflow)
requests_response.close()
break
return response_data
