Exemple #1
0
    def test_two_headers_eq(self):

        self.assertEqual(MyKissHeadersFromStringTest.headers,
                         parse_it(RAW_HEADERS))

        self.assertNotEqual(MyKissHeadersFromStringTest.headers,
                            parse_it(RAW_HEADERS_MOZILLA))
Exemple #2
0
    def test_json_without_indent(self):
        headers: Headers = parse_it(RAW_HEADERS)
        json_headers: str = dumps(headers)

        decoded_headers: Headers = parse_it(json_headers)

        self.assertEqual(
            headers,
            decoded_headers,
            msg="Headers --> json encode --> parse_it --> should be equal",
        )
    def test_get_polymorphic(self):

        headers = parse_it("""accept-ch: DPR
accept-ch-lifetime: 2592000
alt-svc: quic=":443"; ma=2592000; v="46,43", h3-Q050=":443"; ma=2592000, h3-Q049=":443"; ma=2592000, h3-Q048=":443"; ma=2592000, h3-Q046=":443"; ma=2592000, h3-Q043=":443"; ma=2592000
cache-control: private, max-age=0
content-encoding: br
content-length: 64032
content-type: text/html; charset=UTF-8
date: Mon, 16 Mar 2020 21:27:31 GMT
expires: -1
p3p: CP="This is not a P3P policy! See g.co/p3phelp for more info."
server: gws
set-cookie: 1P_JAR=2020-03-16-21; expires=Wed, 15-Apr-2020 21:27:31 GMT; path=/; domain=.google.fr; Secure; SameSite=none
set-cookie: NID=200=IGpBMMA3G7tki0niFFATFQ2BnsNceVP6XBtwOutoyw97AJ4_YFT5l1oLfLeX22xeI_STiP4omAB4rmMP3Sxgyo287ldQGwdZSdPOOZ_Md3roDOMAOtXEQ_hFbUvo0VPjS2gL1y00_6kQwpVxCghI2Ozrx-A4Xks3ZIXRj11RsWs; expires=Tue, 15-Sep-2020 21:27:31 GMT; path=/; domain=.google.fr; Secure; HttpOnly; SameSite=none
set-cookie: CONSENT=WP.284b10; expires=Fri, 01-Jan-2038 00:00:00 GMT; path=/; domain=.google.fr
status: 200""")

        content_type = get_polymorphic(headers, ContentType)

        self.assertEqual("UTF-8", content_type.get_charset())

        self.assertEqual("text/html", content_type.get_mime())

        content_type = get_polymorphic(headers.content_type, ContentType)

        self.assertEqual("UTF-8", content_type.get_charset())

        self.assertEqual("text/html", content_type.get_mime())

        with self.assertRaises(TypeError):
            content_type = get_polymorphic(headers.content_type, Allow)
    def test_httpbin_raw_headers(self):

        headers = parse_it("""Host: developer.mozilla.org
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:50.0) Gecko/20100101 Firefox/50.0
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Language: en-US,en;q=0.5
Accept-Encoding: gzip, deflate, br
Referer: https://developer.mozilla.org/testpage.html
Connection: keep-alive
Upgrade-Insecure-Requests: 1
If-Modified-Since: Mon, 18 Jul 2016 02:36:04 GMT
If-None-Match: "c561c68d0ba92bbeb8b0fff2a9199f722e3a621a"
Cache-Control: max-age=0""".encode("utf-8"))

        self.assertEqual(17, len(headers))

        self.assertEqual("c561c68d0ba92bbeb8b0fff2a9199f722e3a621a",
                         headers.if_none_match)

        self.assertIn("text/html", headers.accept[0])

        self.assertNotIn("text/htm", headers.accept[0])

        self.assertIn("q", headers.accept[-1])

        self.assertEqual("0", headers.cache_control.max_age)
def parseXML(XMLpath):
    allParameters = []

    tree = ET.parse(XMLpath)
    root = tree.getroot()

    for item in root.findall('item'):
        path = item.find('path').text
        request = item.find('request').text
        headers = parse_it(request)
        body = request.split('\n\n')[-1]

        allParameters.extend(getPathParameters(path))

        try:
            contentType = headers['Content-Type']

            if "application/json" in contentType:
                allParameters.extend(getJsonParameters(body))
            elif "application/x-www-form-urlencoded" in contentType:
                allParameters.extend(getFormParameters(body))
        except Exception as e:
            pass

    return allParameters
Exemple #6
0
    def test_verify_autocompletion_capability(self):
        headers = parse_it(RAW_HEADERS_MOZILLA)

        self.assertIn("accept_encoding", dir(headers))

        self.assertIn("accept_language", dir(headers))

        self.assertIn("q", dir(headers.accept))
Exemple #7
0
    def test_control_first_line_not_header(self):

        headers = parse_it(RAW_HEADERS_MOZILLA)

        self.assertEqual(11, len(headers))

        self.assertIn("host", headers)

        self.assertIn("Cache-Control", headers)
Exemple #8
0
    def test_reserved_header_name_keyword(self):
        headers = parse_it(
            "From: Ousret; origin=www.github.com\nIS: 1\nWhile: Not-True")

        self.assertIn("From", headers)

        self.assertEqual("Ousret; origin=www.github.com", headers.from_)

        self.assertIn("Ousret", headers.from_)

        self.assertEqual("Not-True", headers.while_)
    def test_sub(self):
        headers = parse_it(
            """X-My-Testing: 1\nX-My-Second-Test: 1\nX-My-Second-Test: Precisly\nReceived: outpost\nReceived: outpost"""
        )

        self.assertEqual(5, len(headers))

        headers_two = headers - "X-My-Testing"

        self.assertEqual(5, len(headers))

        self.assertEqual(4, len(headers_two))
    def test_parse_response(self):

        headers = parse_it(MyHttpTestKissHeaders.HTTPBIN_GET)

        self.assertEqual(headers.content_type, "application/json")

        self.assertIn("application/json", headers.content_type)

        self.assertNotIn("charset", headers.content_type)

        with self.assertRaises(AttributeError):
            headers.user_agent
    def test_by_setattr(self):
        headers = parse_it(
            """X-My-Testing: 1\nX-My-Second-Test: 1\nX-My-Second-Test: Precisly\nReceived: outpost\nReceived: outpost"""
        )

        self.assertEqual(5, len(headers))

        headers.content_type = "application/json; charset=utf-8"

        self.assertEqual(6, len(headers))

        self.assertEqual("utf-8", headers.content_type.charset)
    def test_remove_by_del_attr(self):
        headers = parse_it(
            """X-My-Testing: 1\nX-My-Second-Test: 1\nX-My-Second-Test: Precisly\nReceived: outpost\nReceived: outpost"""
        )

        self.assertEqual(5, len(headers))

        del headers.x_my_second_test

        self.assertEqual(3, len(headers))

        self.assertNotIn("X-My-Second-Test", headers)
Exemple #13
0
    def test_encode_decode(self):
        response: Response = get("https://www.httpbin.org/get")
        headers: Headers = parse_it(response)

        encoded_headers = encode(headers)
        decoded_headers = decode(encoded_headers)

        self.assertEqual(
            headers,
            decoded_headers,
            msg=
            "Headers --> fn encode() --> fn decode() --> should be equal to initial object",
        )
    def test_add(self):
        headers = parse_it(
            """X-My-Testing: 1\nX-My-Second-Test: 1\nX-My-Second-Test: Precisly\nReceived: outpost\nReceived: outpost"""
        )

        self.assertEqual(5, len(headers))

        headers_ = headers + Header("content-type", "application/json")

        self.assertEqual(6, len(headers_))

        self.assertIn("content-type", headers_)

        self.assertNotIn("content-type", headers)
Exemple #15
0
    def test_fixed_type_output(self):
        headers = parse_it(RAW_HEADERS_MOZILLA)

        self.assertEqual(Header, type(headers.accept))

        lock_output_type()

        self.assertEqual(list, type(headers.accept))

        self.assertEqual(1, len(headers.accept))

        lock_output_type(False)

        self.assertEqual(Header, type(headers.accept))
    def test_httpbin_freeform(self):

        response_headers = MyHttpTestKissHeaders.HTTPBIN_RESPONSE_HEADER.headers
        headers = parse_it(response_headers)

        self.assertIn("freeform", headers)

        self.assertEqual(
            {
                "application/kiss": None,
                "format": "flowed",
                "expires": "Thu, 12 Mar 2020 03:18:25 -0700 (PDT)",
            },
            dict(headers.freeform),
        )

        self.assertIn("application/kiss", headers.freeform)
    def test_replicate_raw_from_objects(self):

        headers = (
            Header("Accept-Ch", "DPR") +
            Header("Accept-Ch-Lifetime", "2592000") +
            AltSvc("quic", ":443", max_age=2592000, versions=["46", "43"]) +
            AltSvc("h3-Q050", ":443", max_age=2592000) +
            AltSvc("h3-Q049", ":443", max_age=2592000) +
            AltSvc("h3-Q048", ":443", max_age=2592000) +
            AltSvc("h3-Q046", ":443", max_age=2592000) +
            AltSvc("h3-Q043", ":443", max_age=2592000) +
            CacheControl("private") + CacheControl(max_age=0) +
            ContentEncoding("br") + ContentLength(64032) +
            ContentType("text/html", charset="utf-8") +
            Date("Mon, 16 Mar 2020 21:27:31 GMT") + Expires("-1") + Header(
                "P3P",
                'CP="This is not a P3P policy! See g.co/p3phelp for more info."'
            ) + Server("gws") + SetCookie(
                "1P_JAR",
                "2020-03-16-21",
                expires="Wed, 15-Apr-2020 21:27:31 GMT",
                path="/",
                domain=".google.fr",
                samesite="none",
                is_secure=True,
                is_httponly=False,
            ) + SetCookie(
                "NID",
                "200=IGpBMMA3G7tki0niFFATFQ2BnsNceVP6XBtwOutoyw97AJ4_YFT5l1oLfLeX22xeI_STiP4omAB4rmMP3Sxgyo287ldQGwdZSdPOOZ_Md3roDOMAOtXEQ_hFbUvo0VPjS2gL1y00_6kQwpVxCghI2Ozrx-A4Xks3ZIXRj11RsWs",
                expires="Tue, 15-Sep-2020 21:27:31 GMT",
                path="/",
                domain=".google.fr",
                samesite="none",
                is_secure=True,
            ) + SetCookie(
                "CONSENT",
                "WP.284b10",
                expires="Fri, 01-Jan-2038 00:00:00 GMT",
                path="/",
                domain=".google.fr",
                is_httponly=False,
            ) + Header("Status", "200") +
            StrictTransportSecurity(max_age=31536000) +
            XFrameOptions("SAMEORIGIN") + XXssProtection(False))

        self.assertEqual(parse_it(RAW_HEADERS), headers)
    def test_replicate_raw_from_objects_request(self):

        headers = (Host("developer.mozilla.org") + UserAgent(
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:50.0) Gecko/20100101 Firefox/50.0"
        ) + Accept("text/html") + Accept("application/xhtml+xml") +
                   Accept("application/xml", qualifier=0.9) +
                   Accept(qualifier=0.8) + AcceptLanguage("en-US") +
                   AcceptLanguage("en", qualifier=0.5) +
                   AcceptEncoding("gzip") + AcceptEncoding("deflate") +
                   AcceptEncoding("br") +
                   Referer("https://developer.mozilla.org/testpage.html") +
                   Connection(True) + UpgradeInsecureRequests() +
                   IfModifiedSince("Mon, 18 Jul 2016 02:36:04 GMT") +
                   IfNoneMatch("c561c68d0ba92bbeb8b0fff2a9199f722e3a621a") +
                   CacheControl(max_age=0))

        self.assertEqual(parse_it(RAW_HEADERS_MOZILLA), headers)
Exemple #19
0
async def async_rotate_fetch(
        url: str,
        protocol: ProxyProto = 'socks5',
        return_proxy: Boolean = False) -> Union[Data, Tuple[Data, Data]]:
    """
	Rotate proxies and perform a GET request. Returns a Data object of `response.status_code`, `response.text`, and
	`response.headers`.
		:param url: URL to fetch.
		:param protocol: 'socks5' or 'http'; must correspond to your ActProxy proxies' type.
		:param return_proxy: Boolean; Return tuple(proxy, ProxyConnector) instead of just ProxyConnector.
		:return: A Data object of the response with resp.text, resp.status_code, and resp.headers
	"""
    url_parts = urlparse(url)
    url_port = 443 if url_parts.scheme == 'https' else 80
    url_path = url_parts.path
    actproxy = one_hot_proxy()

    if proxy := AsyncProxy.from_url(
            f'{protocol}://{actproxy.username}:{actproxy.password}@{actproxy.host}:{actproxy.port}'
    ):
        sock = await proxy.connect(dest_host=url_parts.hostname,
                                   dest_port=url_port)
        reader, writer = await asyncio.open_connection(
            host=None,
            port=None,
            sock=sock,
            ssl=ssl.create_default_context(),
            server_hostname=url_parts.hostname,
        )
        request = (b'GET ' + url_path.encode() + b' HTTP/1.1\r\n'
                   b'Host: ' + url_parts.hostname.encode() + b'\r\n'
                   b'Connection: close\r\n\r\n')
        writer.write(request)
        _resp = await reader.read(-1)
        body = _resp.decode('UTF-8').split('\r\n\r\n', 1)[-1]
        status_code = int(
            _resp.decode('UTF-8').split('\r\n', 1)[0].split(' ')[1])
        headers = parse_it(_resp)
        response_data = Data(text=body,
                             status_code=status_code,
                             headers=headers)
        if return_proxy:
            return response_data, actproxy
        else:
            return response_data
Exemple #20
0
def cache_http_file(uri: str) -> Optional[Gio.File]:
    with requests.get(uri, timeout=5, stream=True) as resp:
        h = parse_it(resp)
        if not str(h.content_type).startswith('image/'):
            return
        ext = mimetypes.guess_extension(str(h.content_type or '')) or ''
        # Is an image, guess filename
        if 'Content-Disposition' in h:
            filename = h.content_disposition.filename
        else:
            path = Path(urlsplit(uri).path)
            filename = path.name
        dirpath = tempfile.mkdtemp(prefix=SHORT_NAME)
        filepath = Path(dirpath) / (filename + ext)
        with open(filepath, 'wb') as f:
            for chunk in resp.iter_content():
                f.write(chunk)
    return Gio.File.new_for_path(str(filepath))
Exemple #21
0
 def process_args(self):
     u = yarl.URL(self.url)
     # Clean fragment, if exist
     url = str(u.with_fragment(None).with_query(None))
     # Strip leading "http://" to be short
     self._url = url[7:] if u.scheme == 'http' else url
     self._params = deque(u.query.items())
     self._data = deque()
     for dstring in self.data:
         result = parse_post_data(dstring)
         self._data.extend(result.data)
         self._errors.extend(result.errors)
     for dstring in self.data_raw:
         result = parse_post_data(dstring, ignore_at=True)
         self._data.extend(result.data)
         self._errors.extend(result.errors)
     for dstring in self.form:
         result = parse_post_data(dstring)
         self._data.extend(result.data)
         self._errors.extend(result.errors)
     for h in self.header:
         headers = parse_it(h)
         if not headers:
             continue
         if HH.CONTENT_TYPE in headers:
             hx = get_polymorphic(headers, ContentType)
             if hx.get_mime() == HH.CONTENT_TYPE_VALUES.json:
                 self._request_json = True
                 continue
         elif HH.AUTHORIZATION in headers and headers.authorization.content.startswith(
                 'Basic '):
             hx = get_polymorphic(headers, BasicAuthorization)
             self._auth = hx
             continue
         # kiss-header doesn't prevent duplicate, so we have to check ourselve
         # Please note the behavior of kiss-headers: The "Accept-Encoding: gzip, deflate"
         # will be parsed to two Header objects, to get all the "value" side, we have to
         # convert the parse result to dict.
         name = headers[0].pretty_name
         if self._headers.has(name):
             del self._headers[name]
         value = headers.to_dict()[name]
         self._headers += Header(name, value)
def custom_action(packet):
    data = {
        'InterfaceLayer': "None",
        'InternetLayer': "None",
        'TransportLayer': "None",
        'ApplicationLayer': "None"
    }

    d = to_dict(packet, strict=False)
    #dunno why but need to iterate over d.maps which is a [], otherwise the order is f****d up
    ip = d["IP"]["src"]
    time = datetime.now()
    user_agent = None
    try:
        if ip not in unique_ips:
            unique_ips.append(ip)
    except:
        None

    if "Raw" in d and d["Raw"]["load"].startswith(b"GET"):
        try:
            headers = parse_it(d["Raw"]["load"])
            user_agent = headers.user_agent
        except:
            print("Exception!!!!!")

    for layer, key in zip(d.maps, data):
        x = next(iter(layer.items()))  #tuple
        data[key] = str(x[0]) + " " + str(x[1])

    client.execute(f'INSERT INTO {db_name}.{data_table} VALUES', [data])
    if user_agent:
        client.execute(f'INSERT INTO {db_name}.{metrics_table} VALUES',
                       [{
                           "Time": str(time),
                           "Ip": ip,
                           "UserAgent": str(user_agent)
                       }])
    print(f"Unique IP addresses: {unique_ips}, packets: {next(counter) + 1}",
          end='\r')
    return None
    def test_httpbin_freeform(self):

        response_headers = get(
            "https://httpbin.org/response-headers",
            params={
                "freeform":
                'application/kiss; format="flowed"; expires=Thu, 12 Mar 2020 03:18:25 -0700 (PDT)'
            },
        ).headers

        headers = parse_it(response_headers)

        self.assertIn("freeform", headers)

        self.assertEqual(
            {
                "application/kiss": None,
                "format": "flowed",
                "expires": "Thu, 12 Mar 2020 03:18:25 -0700 (PDT)",
            },
            dict(headers.freeform),
        )

        self.assertIn("application/kiss", headers.freeform)
    def test_isub(self):
        headers = parse_it(
            """X-My-Testing: 1\nX-My-Second-Test: 1\nX-My-Second-Test: Precisly\nReceived: outpost\nReceived: outpost"""
        )

        self.assertEqual(5, len(headers))

        headers -= "X-My-Testing"

        self.assertEqual(4, len(headers))

        self.assertNotIn("X-My-Testing", headers)

        headers -= "Received"

        self.assertEqual(2, len(headers))

        self.assertNotIn("Received", headers)

        headers -= Header("X-My-Second-Test", "Precisly")

        self.assertEqual(1, len(headers))

        self.assertIn("X-My-Second-Test", headers)
Exemple #25
0
#!/usr/bin/env python3
# @todoing
##
import sys

request_text = sys.stdin.read().encode("utf8")
## @broken idk why people said this works with Python 3.7, it didn't work for me
# from email.parser import BytesParser
# request_line, headers_alone = request_text.split(b'\r\n', 1)
# headers = BytesParser().parsebytes(headers_alone)
##
from kiss_headers import parse_it, dumps
headers = parse_it(request_text)
print(dumps(headers))
##
Exemple #26
0
 def setUp(self) -> None:
     MyKissHeadersFromStringTest.headers = parse_it(RAW_HEADERS)
Exemple #27
0
    def from_message(message):

        global_headers = parse_it(message)

        subject = str(global_headers.subject)

        bodies = list()
        attachements = list()

        for message_part in message.walk():

            charset_declared = None
            content_transfert_encoding_declared = None

            headers = parse_it(message_part.as_string())

            if 'content-type' in headers and 'charset' in headers.content_type:
                charset_declared = headers.content_type.charset

            if 'content-transfer-encoding' in headers:
                content_transfert_encoding_declared = headers.content_transfer_encoding

            raw_body = message_part.as_string()
            raw_body_lines = raw_body.split('\n')

            concerned_body = "\n".join(
                raw_body_lines[raw_body_lines.index(''):])

            if 'multipart/mixed' in headers.content_type or 'multipart/related' in headers.content_type:

                for sub_message_part in message_part.walk():

                    sub_headers = parse_it(sub_message_part.as_string())

                    sub_raw_body = sub_message_part.as_string()
                    sub_raw_body_lines = sub_raw_body.split('\n')

                    sub_concerned_body = "\n".join(
                        sub_raw_body_lines[sub_raw_body_lines.index(''):])

                    sub_charset_declared = sub_headers.content_type.charset if 'charset' in sub_headers.content_type else None

                    if sub_headers.has(
                            'content-transfer-encoding'
                    ) and 'quoted-printable' in sub_headers.content_transfer_encoding:
                        sub_concerned_body = decodestring(
                            sub_concerned_body).decode(
                                sub_charset_declared
                                if sub_charset_declared else 'utf-8',
                                errors='ignore')

                    if (
                            sub_headers.has("content-disposition") is False or
                            'attachment' not in sub_headers.content_disposition
                    ) and 'text/plain' in sub_headers.content_type or 'text/html' in sub_headers.content_type:

                        bodies.append(
                            MailBody(sub_headers,
                                     str(sub_headers.content_type),
                                     sub_concerned_body))

            elif ('content-disposition' not in headers or 'attachment' not in
                  headers.content_disposition) and headers.content_type.has(
                      'text/plain') or headers.content_type.has('text/html'):

                if headers.has(
                        'content-transfer-encoding'
                ) and 'quoted-printable' in headers.content_transfer_encoding:
                    concerned_body = decodestring(concerned_body).decode(
                        charset_declared if charset_declared else 'utf-8',
                        errors='ignore')

                bodies.append(
                    MailBody(headers, str(headers.content_type),
                             concerned_body))

            elif 'multipart/mixed' not in headers.content_type and 'multipart/related' not in headers.content_type:

                filename = None

                if 'Content-Disposition' in headers and 'filename' in headers.content_disposition:
                    filename = headers.content_disposition.filename
                elif headers.content_type.has('name'):
                    filename = headers.content_type.name

                if filename is not None:
                    attachements.append(
                        MailAttachement(
                            filename, str(headers.content_type), "\n".join(
                                raw_body_lines[raw_body_lines.index(''):])))

        try:
            date_parse = parse(
                message.get('Date')).strftime('%Y/%m/%d %H:%M:%S %z')
        except:
            date_parse = message.get('Date')

        return Mail(message.get('Message-ID'),
                    subject,
                    bodies,
                    attachements,
                    parseaddr(message.get('From')),
                    parseaddr(message.get('To')),
                    raw_content=bytes(message),
                    date_received=date_parse)
Exemple #28
0
    def test_headers_to_bytes(self):
        headers = parse_it(RAW_HEADERS_MOZILLA)

        self.assertEqual(headers, parse_it(bytes(headers)))
    def test_httpx_requests_parse_it_eq(self):

        headers = parse_it(MyHttpTestKissHeaders.HTTPBIN_GET)

        for header in parse_it(MyHttpTestKissHeaders.HTTPBIEN_GET_HTTPX):
            self.assertIn(header.name, headers)
    def test_bytes_headers(self):

        self.assertEqual(MyKissHeadersFromStringTest.headers,
                         parse_it(RAW_HEADERS.encode("utf-8")))