def test_two_headers_eq(self):
self.assertEqual(MyKissHeadersFromStringTest.headers,
parse_it(RAW_HEADERS))
self.assertNotEqual(MyKissHeadersFromStringTest.headers,
parse_it(RAW_HEADERS_MOZILLA))
def test_json_without_indent(self):
headers: Headers = parse_it(RAW_HEADERS)
json_headers: str = dumps(headers)
decoded_headers: Headers = parse_it(json_headers)
self.assertEqual(
headers,
decoded_headers,
msg="Headers --> json encode --> parse_it --> should be equal",
)
def test_get_polymorphic(self):
headers = parse_it("""accept-ch: DPR
accept-ch-lifetime: 2592000
alt-svc: quic=":443"; ma=2592000; v="46,43", h3-Q050=":443"; ma=2592000, h3-Q049=":443"; ma=2592000, h3-Q048=":443"; ma=2592000, h3-Q046=":443"; ma=2592000, h3-Q043=":443"; ma=2592000
cache-control: private, max-age=0
content-encoding: br
content-length: 64032
content-type: text/html; charset=UTF-8
date: Mon, 16 Mar 2020 21:27:31 GMT
expires: -1
p3p: CP="This is not a P3P policy! See g.co/p3phelp for more info."
server: gws
set-cookie: 1P_JAR=2020-03-16-21; expires=Wed, 15-Apr-2020 21:27:31 GMT; path=/; domain=.google.fr; Secure; SameSite=none
set-cookie: NID=200=IGpBMMA3G7tki0niFFATFQ2BnsNceVP6XBtwOutoyw97AJ4_YFT5l1oLfLeX22xeI_STiP4omAB4rmMP3Sxgyo287ldQGwdZSdPOOZ_Md3roDOMAOtXEQ_hFbUvo0VPjS2gL1y00_6kQwpVxCghI2Ozrx-A4Xks3ZIXRj11RsWs; expires=Tue, 15-Sep-2020 21:27:31 GMT; path=/; domain=.google.fr; Secure; HttpOnly; SameSite=none
set-cookie: CONSENT=WP.284b10; expires=Fri, 01-Jan-2038 00:00:00 GMT; path=/; domain=.google.fr
status: 200""")
content_type = get_polymorphic(headers, ContentType)
self.assertEqual("UTF-8", content_type.get_charset())
self.assertEqual("text/html", content_type.get_mime())
content_type = get_polymorphic(headers.content_type, ContentType)
self.assertEqual("UTF-8", content_type.get_charset())
self.assertEqual("text/html", content_type.get_mime())
with self.assertRaises(TypeError):
content_type = get_polymorphic(headers.content_type, Allow)
def test_httpbin_raw_headers(self):
headers = parse_it("""Host: developer.mozilla.org
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:50.0) Gecko/20100101 Firefox/50.0
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Language: en-US,en;q=0.5
Accept-Encoding: gzip, deflate, br
Referer: https://developer.mozilla.org/testpage.html
Connection: keep-alive
Upgrade-Insecure-Requests: 1
If-Modified-Since: Mon, 18 Jul 2016 02:36:04 GMT
If-None-Match: "c561c68d0ba92bbeb8b0fff2a9199f722e3a621a"
Cache-Control: max-age=0""".encode("utf-8"))
self.assertEqual(17, len(headers))
self.assertEqual("c561c68d0ba92bbeb8b0fff2a9199f722e3a621a",
headers.if_none_match)
self.assertIn("text/html", headers.accept[0])
self.assertNotIn("text/htm", headers.accept[0])
self.assertIn("q", headers.accept[-1])
self.assertEqual("0", headers.cache_control.max_age)
def parseXML(XMLpath):
allParameters = []
tree = ET.parse(XMLpath)
root = tree.getroot()
for item in root.findall('item'):
path = item.find('path').text
request = item.find('request').text
headers = parse_it(request)
body = request.split('\n\n')[-1]
allParameters.extend(getPathParameters(path))
try:
contentType = headers['Content-Type']
if "application/json" in contentType:
allParameters.extend(getJsonParameters(body))
elif "application/x-www-form-urlencoded" in contentType:
allParameters.extend(getFormParameters(body))
except Exception as e:
pass
return allParameters
def test_verify_autocompletion_capability(self):
headers = parse_it(RAW_HEADERS_MOZILLA)
self.assertIn("accept_encoding", dir(headers))
self.assertIn("accept_language", dir(headers))
self.assertIn("q", dir(headers.accept))
def test_control_first_line_not_header(self):
headers = parse_it(RAW_HEADERS_MOZILLA)
self.assertEqual(11, len(headers))
self.assertIn("host", headers)
self.assertIn("Cache-Control", headers)
def test_reserved_header_name_keyword(self):
headers = parse_it(
"From: Ousret; origin=www.github.com\nIS: 1\nWhile: Not-True")
self.assertIn("From", headers)
self.assertEqual("Ousret; origin=www.github.com", headers.from_)
self.assertIn("Ousret", headers.from_)
self.assertEqual("Not-True", headers.while_)
def test_sub(self):
headers = parse_it(
"""X-My-Testing: 1\nX-My-Second-Test: 1\nX-My-Second-Test: Precisly\nReceived: outpost\nReceived: outpost"""
)
self.assertEqual(5, len(headers))
headers_two = headers - "X-My-Testing"
self.assertEqual(5, len(headers))
self.assertEqual(4, len(headers_two))
def test_parse_response(self):
headers = parse_it(MyHttpTestKissHeaders.HTTPBIN_GET)
self.assertEqual(headers.content_type, "application/json")
self.assertIn("application/json", headers.content_type)
self.assertNotIn("charset", headers.content_type)
with self.assertRaises(AttributeError):
headers.user_agent
def test_by_setattr(self):
headers = parse_it(
"""X-My-Testing: 1\nX-My-Second-Test: 1\nX-My-Second-Test: Precisly\nReceived: outpost\nReceived: outpost"""
)
self.assertEqual(5, len(headers))
headers.content_type = "application/json; charset=utf-8"
self.assertEqual(6, len(headers))
self.assertEqual("utf-8", headers.content_type.charset)
def test_remove_by_del_attr(self):
headers = parse_it(
"""X-My-Testing: 1\nX-My-Second-Test: 1\nX-My-Second-Test: Precisly\nReceived: outpost\nReceived: outpost"""
)
self.assertEqual(5, len(headers))
del headers.x_my_second_test
self.assertEqual(3, len(headers))
self.assertNotIn("X-My-Second-Test", headers)
def test_encode_decode(self):
response: Response = get("https://www.httpbin.org/get")
headers: Headers = parse_it(response)
encoded_headers = encode(headers)
decoded_headers = decode(encoded_headers)
self.assertEqual(
headers,
decoded_headers,
msg=
"Headers --> fn encode() --> fn decode() --> should be equal to initial object",
)
def test_add(self):
headers = parse_it(
"""X-My-Testing: 1\nX-My-Second-Test: 1\nX-My-Second-Test: Precisly\nReceived: outpost\nReceived: outpost"""
)
self.assertEqual(5, len(headers))
headers_ = headers + Header("content-type", "application/json")
self.assertEqual(6, len(headers_))
self.assertIn("content-type", headers_)
self.assertNotIn("content-type", headers)
def test_fixed_type_output(self):
headers = parse_it(RAW_HEADERS_MOZILLA)
self.assertEqual(Header, type(headers.accept))
lock_output_type()
self.assertEqual(list, type(headers.accept))
self.assertEqual(1, len(headers.accept))
lock_output_type(False)
self.assertEqual(Header, type(headers.accept))
def test_httpbin_freeform(self):
response_headers = MyHttpTestKissHeaders.HTTPBIN_RESPONSE_HEADER.headers
headers = parse_it(response_headers)
self.assertIn("freeform", headers)
self.assertEqual(
{
"application/kiss": None,
"format": "flowed",
"expires": "Thu, 12 Mar 2020 03:18:25 -0700 (PDT)",
},
dict(headers.freeform),
)
self.assertIn("application/kiss", headers.freeform)
def test_replicate_raw_from_objects(self):
headers = (
Header("Accept-Ch", "DPR") +
Header("Accept-Ch-Lifetime", "2592000") +
AltSvc("quic", ":443", max_age=2592000, versions=["46", "43"]) +
AltSvc("h3-Q050", ":443", max_age=2592000) +
AltSvc("h3-Q049", ":443", max_age=2592000) +
AltSvc("h3-Q048", ":443", max_age=2592000) +
AltSvc("h3-Q046", ":443", max_age=2592000) +
AltSvc("h3-Q043", ":443", max_age=2592000) +
CacheControl("private") + CacheControl(max_age=0) +
ContentEncoding("br") + ContentLength(64032) +
ContentType("text/html", charset="utf-8") +
Date("Mon, 16 Mar 2020 21:27:31 GMT") + Expires("-1") + Header(
"P3P",
'CP="This is not a P3P policy! See g.co/p3phelp for more info."'
) + Server("gws") + SetCookie(
"1P_JAR",
"2020-03-16-21",
expires="Wed, 15-Apr-2020 21:27:31 GMT",
path="/",
domain=".google.fr",
samesite="none",
is_secure=True,
is_httponly=False,
) + SetCookie(
"NID",
"200=IGpBMMA3G7tki0niFFATFQ2BnsNceVP6XBtwOutoyw97AJ4_YFT5l1oLfLeX22xeI_STiP4omAB4rmMP3Sxgyo287ldQGwdZSdPOOZ_Md3roDOMAOtXEQ_hFbUvo0VPjS2gL1y00_6kQwpVxCghI2Ozrx-A4Xks3ZIXRj11RsWs",
expires="Tue, 15-Sep-2020 21:27:31 GMT",
path="/",
domain=".google.fr",
samesite="none",
is_secure=True,
) + SetCookie(
"CONSENT",
"WP.284b10",
expires="Fri, 01-Jan-2038 00:00:00 GMT",
path="/",
domain=".google.fr",
is_httponly=False,
) + Header("Status", "200") +
StrictTransportSecurity(max_age=31536000) +
XFrameOptions("SAMEORIGIN") + XXssProtection(False))
self.assertEqual(parse_it(RAW_HEADERS), headers)
def test_replicate_raw_from_objects_request(self):
headers = (Host("developer.mozilla.org") + UserAgent(
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:50.0) Gecko/20100101 Firefox/50.0"
) + Accept("text/html") + Accept("application/xhtml+xml") +
Accept("application/xml", qualifier=0.9) +
Accept(qualifier=0.8) + AcceptLanguage("en-US") +
AcceptLanguage("en", qualifier=0.5) +
AcceptEncoding("gzip") + AcceptEncoding("deflate") +
AcceptEncoding("br") +
Referer("https://developer.mozilla.org/testpage.html") +
Connection(True) + UpgradeInsecureRequests() +
IfModifiedSince("Mon, 18 Jul 2016 02:36:04 GMT") +
IfNoneMatch("c561c68d0ba92bbeb8b0fff2a9199f722e3a621a") +
CacheControl(max_age=0))
self.assertEqual(parse_it(RAW_HEADERS_MOZILLA), headers)
async def async_rotate_fetch(
url: str,
protocol: ProxyProto = 'socks5',
return_proxy: Boolean = False) -> Union[Data, Tuple[Data, Data]]:
"""
Rotate proxies and perform a GET request. Returns a Data object of `response.status_code`, `response.text`, and
`response.headers`.
:param url: URL to fetch.
:param protocol: 'socks5' or 'http'; must correspond to your ActProxy proxies' type.
:param return_proxy: Boolean; Return tuple(proxy, ProxyConnector) instead of just ProxyConnector.
:return: A Data object of the response with resp.text, resp.status_code, and resp.headers
"""
url_parts = urlparse(url)
url_port = 443 if url_parts.scheme == 'https' else 80
url_path = url_parts.path
actproxy = one_hot_proxy()
if proxy := AsyncProxy.from_url(
f'{protocol}://{actproxy.username}:{actproxy.password}@{actproxy.host}:{actproxy.port}'
):
sock = await proxy.connect(dest_host=url_parts.hostname,
dest_port=url_port)
reader, writer = await asyncio.open_connection(
host=None,
port=None,
sock=sock,
ssl=ssl.create_default_context(),
server_hostname=url_parts.hostname,
)
request = (b'GET ' + url_path.encode() + b' HTTP/1.1\r\n'
b'Host: ' + url_parts.hostname.encode() + b'\r\n'
b'Connection: close\r\n\r\n')
writer.write(request)
_resp = await reader.read(-1)
body = _resp.decode('UTF-8').split('\r\n\r\n', 1)[-1]
status_code = int(
_resp.decode('UTF-8').split('\r\n', 1)[0].split(' ')[1])
headers = parse_it(_resp)
response_data = Data(text=body,
status_code=status_code,
headers=headers)
if return_proxy:
return response_data, actproxy
else:
return response_data
def cache_http_file(uri: str) -> Optional[Gio.File]:
with requests.get(uri, timeout=5, stream=True) as resp:
h = parse_it(resp)
if not str(h.content_type).startswith('image/'):
return
ext = mimetypes.guess_extension(str(h.content_type or '')) or ''
# Is an image, guess filename
if 'Content-Disposition' in h:
filename = h.content_disposition.filename
else:
path = Path(urlsplit(uri).path)
filename = path.name
dirpath = tempfile.mkdtemp(prefix=SHORT_NAME)
filepath = Path(dirpath) / (filename + ext)
with open(filepath, 'wb') as f:
for chunk in resp.iter_content():
f.write(chunk)
return Gio.File.new_for_path(str(filepath))
def process_args(self):
u = yarl.URL(self.url)
# Clean fragment, if exist
url = str(u.with_fragment(None).with_query(None))
# Strip leading "http://" to be short
self._url = url[7:] if u.scheme == 'http' else url
self._params = deque(u.query.items())
self._data = deque()
for dstring in self.data:
result = parse_post_data(dstring)
self._data.extend(result.data)
self._errors.extend(result.errors)
for dstring in self.data_raw:
result = parse_post_data(dstring, ignore_at=True)
self._data.extend(result.data)
self._errors.extend(result.errors)
for dstring in self.form:
result = parse_post_data(dstring)
self._data.extend(result.data)
self._errors.extend(result.errors)
for h in self.header:
headers = parse_it(h)
if not headers:
continue
if HH.CONTENT_TYPE in headers:
hx = get_polymorphic(headers, ContentType)
if hx.get_mime() == HH.CONTENT_TYPE_VALUES.json:
self._request_json = True
continue
elif HH.AUTHORIZATION in headers and headers.authorization.content.startswith(
'Basic '):
hx = get_polymorphic(headers, BasicAuthorization)
self._auth = hx
continue
# kiss-header doesn't prevent duplicate, so we have to check ourselve
# Please note the behavior of kiss-headers: The "Accept-Encoding: gzip, deflate"
# will be parsed to two Header objects, to get all the "value" side, we have to
# convert the parse result to dict.
name = headers[0].pretty_name
if self._headers.has(name):
del self._headers[name]
value = headers.to_dict()[name]
self._headers += Header(name, value)
def custom_action(packet):
data = {
'InterfaceLayer': "None",
'InternetLayer': "None",
'TransportLayer': "None",
'ApplicationLayer': "None"
}
d = to_dict(packet, strict=False)
#dunno why but need to iterate over d.maps which is a [], otherwise the order is f****d up
ip = d["IP"]["src"]
time = datetime.now()
user_agent = None
try:
if ip not in unique_ips:
unique_ips.append(ip)
except:
None
if "Raw" in d and d["Raw"]["load"].startswith(b"GET"):
try:
headers = parse_it(d["Raw"]["load"])
user_agent = headers.user_agent
except:
print("Exception!!!!!")
for layer, key in zip(d.maps, data):
x = next(iter(layer.items())) #tuple
data[key] = str(x[0]) + " " + str(x[1])
client.execute(f'INSERT INTO {db_name}.{data_table} VALUES', [data])
if user_agent:
client.execute(f'INSERT INTO {db_name}.{metrics_table} VALUES',
[{
"Time": str(time),
"Ip": ip,
"UserAgent": str(user_agent)
}])
print(f"Unique IP addresses: {unique_ips}, packets: {next(counter) + 1}",
end='\r')
return None
def test_httpbin_freeform(self):
response_headers = get(
"https://httpbin.org/response-headers",
params={
"freeform":
'application/kiss; format="flowed"; expires=Thu, 12 Mar 2020 03:18:25 -0700 (PDT)'
},
).headers
headers = parse_it(response_headers)
self.assertIn("freeform", headers)
self.assertEqual(
{
"application/kiss": None,
"format": "flowed",
"expires": "Thu, 12 Mar 2020 03:18:25 -0700 (PDT)",
},
dict(headers.freeform),
)
self.assertIn("application/kiss", headers.freeform)
def test_isub(self):
headers = parse_it(
"""X-My-Testing: 1\nX-My-Second-Test: 1\nX-My-Second-Test: Precisly\nReceived: outpost\nReceived: outpost"""
)
self.assertEqual(5, len(headers))
headers -= "X-My-Testing"
self.assertEqual(4, len(headers))
self.assertNotIn("X-My-Testing", headers)
headers -= "Received"
self.assertEqual(2, len(headers))
self.assertNotIn("Received", headers)
headers -= Header("X-My-Second-Test", "Precisly")
self.assertEqual(1, len(headers))
self.assertIn("X-My-Second-Test", headers)
#!/usr/bin/env python3
# @todoing
##
import sys
request_text = sys.stdin.read().encode("utf8")
## @broken idk why people said this works with Python 3.7, it didn't work for me
# from email.parser import BytesParser
# request_line, headers_alone = request_text.split(b'\r\n', 1)
# headers = BytesParser().parsebytes(headers_alone)
##
from kiss_headers import parse_it, dumps
headers = parse_it(request_text)
print(dumps(headers))
##
def setUp(self) -> None:
MyKissHeadersFromStringTest.headers = parse_it(RAW_HEADERS)
def from_message(message):
global_headers = parse_it(message)
subject = str(global_headers.subject)
bodies = list()
attachements = list()
for message_part in message.walk():
charset_declared = None
content_transfert_encoding_declared = None
headers = parse_it(message_part.as_string())
if 'content-type' in headers and 'charset' in headers.content_type:
charset_declared = headers.content_type.charset
if 'content-transfer-encoding' in headers:
content_transfert_encoding_declared = headers.content_transfer_encoding
raw_body = message_part.as_string()
raw_body_lines = raw_body.split('\n')
concerned_body = "\n".join(
raw_body_lines[raw_body_lines.index(''):])
if 'multipart/mixed' in headers.content_type or 'multipart/related' in headers.content_type:
for sub_message_part in message_part.walk():
sub_headers = parse_it(sub_message_part.as_string())
sub_raw_body = sub_message_part.as_string()
sub_raw_body_lines = sub_raw_body.split('\n')
sub_concerned_body = "\n".join(
sub_raw_body_lines[sub_raw_body_lines.index(''):])
sub_charset_declared = sub_headers.content_type.charset if 'charset' in sub_headers.content_type else None
if sub_headers.has(
'content-transfer-encoding'
) and 'quoted-printable' in sub_headers.content_transfer_encoding:
sub_concerned_body = decodestring(
sub_concerned_body).decode(
sub_charset_declared
if sub_charset_declared else 'utf-8',
errors='ignore')
if (
sub_headers.has("content-disposition") is False or
'attachment' not in sub_headers.content_disposition
) and 'text/plain' in sub_headers.content_type or 'text/html' in sub_headers.content_type:
bodies.append(
MailBody(sub_headers,
str(sub_headers.content_type),
sub_concerned_body))
elif ('content-disposition' not in headers or 'attachment' not in
headers.content_disposition) and headers.content_type.has(
'text/plain') or headers.content_type.has('text/html'):
if headers.has(
'content-transfer-encoding'
) and 'quoted-printable' in headers.content_transfer_encoding:
concerned_body = decodestring(concerned_body).decode(
charset_declared if charset_declared else 'utf-8',
errors='ignore')
bodies.append(
MailBody(headers, str(headers.content_type),
concerned_body))
elif 'multipart/mixed' not in headers.content_type and 'multipart/related' not in headers.content_type:
filename = None
if 'Content-Disposition' in headers and 'filename' in headers.content_disposition:
filename = headers.content_disposition.filename
elif headers.content_type.has('name'):
filename = headers.content_type.name
if filename is not None:
attachements.append(
MailAttachement(
filename, str(headers.content_type), "\n".join(
raw_body_lines[raw_body_lines.index(''):])))
try:
date_parse = parse(
message.get('Date')).strftime('%Y/%m/%d %H:%M:%S %z')
except:
date_parse = message.get('Date')
return Mail(message.get('Message-ID'),
subject,
bodies,
attachements,
parseaddr(message.get('From')),
parseaddr(message.get('To')),
raw_content=bytes(message),
date_received=date_parse)
def test_headers_to_bytes(self):
headers = parse_it(RAW_HEADERS_MOZILLA)
self.assertEqual(headers, parse_it(bytes(headers)))
def test_httpx_requests_parse_it_eq(self):
headers = parse_it(MyHttpTestKissHeaders.HTTPBIN_GET)
for header in parse_it(MyHttpTestKissHeaders.HTTPBIEN_GET_HTTPX):
self.assertIn(header.name, headers)
def test_bytes_headers(self):
self.assertEqual(MyKissHeadersFromStringTest.headers,
parse_it(RAW_HEADERS.encode("utf-8")))