def parse_ns_headers(ns_headers): """Ad-hoc parser for Netscape protocol cookie-attributes. The old Netscape cookie format for Set-Cookie can for instance contain an unquoted "," in the expires field, so we have to use this ad-hoc parser instead of split_header_words. XXX This may not make the best possible effort to parse all the crap that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient parser is probably better, so could do worse than following that if this ever gives any trouble. Currently, this is also used for parsing RFC 2109 cookies. """ known_attrs = ( "expires", "domain", "path", "secure", # RFC 2109 attrs (may turn up in Netscape cookies, too) "version", "port", "max-age") result = [] for ns_header in ns_headers: pairs = [] version_set = False params = re.split(r";\s*", ns_header) for ii in range(len(params)): param = params[ii] param = param.rstrip() if param == "": continue if "=" not in param: k, v = param, None else: k, v = re.split(r"\s*=\s*", param, 1) k = k.lstrip() if ii != 0: lc = k.lower() if lc in known_attrs: k = lc if k == "version": # This is an RFC 2109 cookie. v = strip_quotes(v) version_set = True if k == "expires": # convert expires date to seconds since epoch v = http2time(strip_quotes(v)) # None if invalid pairs.append((k, v)) if pairs: if not version_set: pairs.append(("version", "0")) result.append(pairs) return result
def test_set_cookie(): cookiejar = FileCookieJar() _cookie = {"value_0": "v_0", "value_1": "v_1", "value_2": "v_2"} c = SimpleCookie(_cookie) domain_0 = ".test_domain" domain_1 = "test_domain" max_age = "09 Feb 1994 22:23:32 GMT" expires = http2time(max_age) path = "test/path" c["value_0"]["max-age"] = max_age c["value_0"]["domain"] = domain_0 c["value_0"]["path"] = path c["value_1"]["domain"] = domain_1 util.set_cookie(cookiejar, c) cookies = cookiejar._cookies c_0 = cookies[domain_0][path]["value_0"] c_1 = cookies[domain_1][""]["value_1"] c_2 = cookies[""][""]["value_2"] assert not (c_2.domain_specified and c_2.path_specified) assert c_1.domain_specified and not c_1.domain_initial_dot and not \ c_1.path_specified assert c_0.domain_specified and c_0.domain_initial_dot and \ c_0.path_specified assert c_0.expires == expires assert c_0.domain == domain_0 assert c_0.name == "value_0" assert c_0.path == path assert c_0.value == "v_0" assert not c_1.expires assert c_1.domain == domain_1 assert c_1.name == "value_1" assert c_1.path == "" assert c_1.value == "v_1" assert not c_2.expires assert c_2.domain == "" assert c_2.name == "value_2" assert c_2.path == "" assert c_2.value == "v_2"
def test_set_cookie(): cookiejar = FileCookieJar() _cookie = {"value_0": "v_0", "value_1": "v_1", "value_2": "v_2"} c = SimpleCookie(_cookie) domain_0 = ".test_domain" domain_1 = "test_domain" max_age = "09 Feb 1994 22:23:32 GMT" expires = http2time(max_age) path = "test/path" c["value_0"]["max-age"] = max_age c["value_0"]["domain"] = domain_0 c["value_0"]["path"] = path c["value_1"]["domain"] = domain_1 util.set_cookie(cookiejar, c) cookies = cookiejar._cookies c_0 = cookies[domain_0][path]["value_0"] c_1 = cookies[domain_1][""]["value_1"] c_2 = cookies[""][""]["value_2"] assert not (c_2.domain_specified and c_2.path_specified) assert c_1.domain_specified and not c_1.domain_initial_dot and not c_1.path_specified assert c_0.domain_specified and c_0.domain_initial_dot and c_0.path_specified assert c_0.expires == expires assert c_0.domain == domain_0 assert c_0.name == "value_0" assert c_0.path == path assert c_0.value == "v_0" assert not c_1.expires assert c_1.domain == domain_1 assert c_1.name == "value_1" assert c_1.path == "" assert c_1.value == "v_1" assert not c_2.expires assert c_2.domain == "" assert c_2.name == "value_2" assert c_2.path == "" assert c_2.value == "v_2"
def make(self, cookie_string): # split "Set-Cookie:x=y; domain=...; expires=...;..." set_string, tuple_string = cookie_string.split(":", 1) # parse version from set string version = self._version(set_string) # parse name, value from tuple string nv = self._name_value(tuple_string) # change tuple string to dict dict = self._dict(tuple_string) if nv is None or dict is None: return None name, value = nv port = dict.get("port", None) port_specified = port is not None domain = dict.get("domain", None) domain_specified = domain is not None domain_initial_dot = False if domain is not None: domain_initial_dot = domain.startswith(".") path = dict.get("path", None) path_specified = path is not None secure = dict.get("secure", False) expires = dict.get("expires", None) if expires is not None: expires = http2time(expires) discard = dict.get("discard", False) comment = dict.get("comment", None) comment_url = None rest = {} # create cookielib.Cookie object cookie = Cookie(version, name, value, port, port_specified, domain, domain_specified, domain_initial_dot, path, path_specified, secure, expires, discard, comment, comment_url, rest) return cookie
def set_cookie(cookiejar, kaka): """PLaces a cookie (a cookielib.Cookie based on a set-cookie header line) in the cookie jar. Always chose the shortest expires time. :param cookiejar: :param kaka: Cookie """ # default rfc2109=False # max-age, httponly for cookie_name, morsel in kaka.items(): std_attr = ATTRS.copy() std_attr["name"] = cookie_name _tmp = morsel.coded_value if _tmp.startswith('"') and _tmp.endswith('"'): std_attr["value"] = _tmp[1:-1] else: std_attr["value"] = _tmp std_attr["version"] = 0 attr = "" # copy attributes that have values try: for attr in morsel.keys(): if attr in ATTRS: if morsel[attr]: if attr == "expires": std_attr[attr] = http2time(morsel[attr]) else: std_attr[attr] = morsel[attr] elif attr == "max-age": if morsel[attr]: std_attr["expires"] = http2time(morsel[attr]) except TimeFormatError: # Ignore cookie logger.info( "Time format error on %s parameter in received cookie" % (sanitize(attr), )) continue for att, spec in PAIRS.items(): if std_attr[att]: std_attr[spec] = True if std_attr["domain"] and std_attr["domain"].startswith("."): std_attr["domain_initial_dot"] = True if morsel["max-age"] is 0: try: cookiejar.clear(domain=std_attr["domain"], path=std_attr["path"], name=std_attr["name"]) except ValueError: pass else: # Fix for Microsoft cookie error if "version" in std_attr: try: std_attr["version"] = std_attr["version"].split(",")[0] except (TypeError, AttributeError): pass new_cookie = Cookie(**std_attr) cookiejar.set_cookie(new_cookie)
def set_cookie(cookiejar, kaka): """PLaces a cookie (a http_cookielib.Cookie based on a set-cookie header line) in the cookie jar. Always chose the shortest expires time. :param cookiejar: :param kaka: Cookie """ # default rfc2109=False # max-age, httponly for cookie_name, morsel in kaka.items(): std_attr = ATTRS.copy() std_attr["name"] = cookie_name _tmp = morsel.coded_value if _tmp.startswith('"') and _tmp.endswith('"'): std_attr["value"] = _tmp[1:-1] else: std_attr["value"] = _tmp std_attr["version"] = 0 attr = "" # copy attributes that have values try: for attr in morsel.keys(): if attr in ATTRS: if morsel[attr]: if attr == "expires": std_attr[attr] = http2time(morsel[attr]) else: std_attr[attr] = morsel[attr] elif attr == "max-age": if morsel[attr]: std_attr["expires"] = http2time(morsel[attr]) except TimeFormatError: # Ignore cookie logger.info( "Time format error on %s parameter in received cookie" % ( sanitize(attr),)) continue for att, spec in PAIRS.items(): if std_attr[att]: std_attr[spec] = True if std_attr["domain"] and std_attr["domain"].startswith("."): std_attr["domain_initial_dot"] = True if morsel["max-age"] == 0: try: cookiejar.clear(domain=std_attr["domain"], path=std_attr["path"], name=std_attr["name"]) except ValueError: pass else: # Fix for Microsoft cookie error if "version" in std_attr: try: std_attr["version"] = std_attr["version"].split(",")[0] except (TypeError, AttributeError): pass new_cookie = http_cookiejar.Cookie(**std_attr) cookiejar.set_cookie(new_cookie)
def _http(self, url, post=None, headers={}, method=None, proxy=None, cookcookie=True, location=True, locationcount=0): if not method: if post: method = "POST" else: method = "GET" rep = None urlinfo = https, host, port, path = self._get_urlinfo(url) log = {} con = self.conpool._get_connect(urlinfo, proxy) # con .set_debuglevel(2) #? conerr = False try: con._send_output = self._send_output(con._send_output, con, log) tmpheaders = copy.deepcopy(headers) tmpheaders['Accept-Encoding'] = 'gzip, deflate' tmpheaders['Connection'] = 'Keep-Alive' tmpheaders[ 'User-Agent'] = tmpheaders['User-Agent'] if tmpheaders.get( 'User-Agent' ) else 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36' if cookcookie: c = self.cookiepool.get(host, None) if not c: self.cookiepool[host] = self.initcookie c = self.cookiepool.get(host) if 'Cookie' in tmpheaders: cookie_str = tmpheaders['Cookie'].strip() if not cookie_str.endswith(';'): cookie_str += ";" for cookiepart in cookie_str.split(";"): if cookiepart.strip() != "": cookiekey, cookievalue = cookiepart.split("=", 1) c[cookiekey.strip()] = cookievalue.strip() for k in c.keys(): m = c[k] # check cookie path if path.find(m['path']) != 0: continue expires = m['expires'] if not expires: continue # check cookie expires time if cookiejar.http2time(expires) < time.time(): del c[k] cookie_str = c.output(attrs=[], header='', sep=';').strip() if cookie_str: tmpheaders['Cookie'] = cookie_str if post: tmpheaders['Content-Type'] = tmpheaders.get( 'Content-Type', 'application/x-www-form-urlencoded') else: # content-length err 411 tmpheaders['Content-Length'] = tmpheaders.get( 'Content-Length', 0) if method == 'GET': del tmpheaders['Content-Length'] con.request(method, path, post, tmpheaders) rep = con.getresponse() body = rep.read() encode = rep.msg.get('content-encoding', None) if encode == 'gzip': body = gzip.GzipFile(fileobj=StringIO.StringIO(body)).read() elif encode == 'deflate': try: body = zlib.decompress(body, -zlib.MAX_WBITS) except: body = zlib.decompress(body) body = self._decode_html(rep.msg.dict.get('content-type', ''), body) retheader = Compatibleheader(str(rep.msg)) retheader.setdict(rep.msg.dict) redirect = rep.msg.dict.get('location', url) if not redirect.startswith('http'): redirect = urlparse.urljoin(url, redirect) if cookcookie and "set-cookie" in rep.msg.dict: c = self.cookiepool[host] c.load(rep.msg.dict['set-cookie']) except http.client.ImproperConnectionState: conerr = True raise except: raise finally: if conerr or (rep and rep.msg.get('connection') == 'close') or proxy: self.conpool._release_connect(urlinfo) con.close() else: self.conpool._put_connect(urlinfo, con) log["url"] = url if post: log['request'] += "\r\n\r\n" + post log["response"] = "HTTP/%.1f %d %s" % ( rep.version * 0.1, rep.status, rep.reason) + '\r\n' + str(retheader) + '\r\n' + (body[:4096]) if location and url != redirect and locationcount < 5: method = 'HEAD' if method == 'HEAD' else 'GET' a, b, c, d, e = self._http(redirect, method=method, proxy=proxy, cookcookie=cookcookie, location=location, locationcount=locationcount + 1) log["response"] = e["response"] return a, b, c, d, log return rep.status, retheader, body, redirect, log
def test_ratelimiting(self): """Test actually being ratelimited.""" # bypass auth and make all verifications successful session.post(API_ROOT + '/debug/3') set_session(0) # set up a ratelimit that is easily testable session.post(API_ROOT + '/admin/ratelimits/kenny2scratch', json={'ratelimit': 2}) resp = session.put(API_ROOT + '/verify/deathly_hallows') now = int(http2time(resp.headers['Date'])) # anything that calls the verify API returns a X-Requests-Remaining # header like left=2, resets=1598514504, to=2 self.assertIn('X-Requests-Remaining', resp.headers, 'no custom header') # *starting* verification should not trigger ratelimits self.assertEqual(resp.headers['X-Requests-Remaining'], HEADER_FORMAT.format(2, now + LIMIT_PER_TIME, 2), 'incorrect header format or values') time.sleep(2) # the reset time should always be LIMIT_PER_TIME seconds into the future # up until "left" is less than "to" resp = session.post(API_ROOT + '/verify/deathly_hallows') now = int(http2time(resp.headers['Date'])) # the POST API should return it too self.assertIn('X-Requests-Remaining', resp.headers, 'no custom header') # *finishing* verification should decrement "left" self.assertEqual( resp.headers['X-Requests-Remaining'], HEADER_FORMAT.format(1, now + LIMIT_PER_TIME, 2), 'incorrect header format or values, most likely ' 'left is not 1') time.sleep(2) resp = session.put(API_ROOT + '/verify/deathly_hallows') # now that left < to, resets should still be LIMIT_PER_TIME after the # "now" that was *before* the last POST, rather than LIMIT_PER_TIME # after *this* "now". Also, this shouldn't decrement "left" self.assertEqual( resp.headers['X-Requests-Remaining'], HEADER_FORMAT.format(1, now + LIMIT_PER_TIME, 2), 'incorrect header format or values, most likely ' 'left is not 1 or resets changed since last request') # don't pretend it worked session.post(API_ROOT + '/debug/1') resp = session.post(API_ROOT + '/verify/deathly_hallows') self.assertEqual(resp.status_code, 403, "wait, didn't fail?") # even on failed verification, the API was called self.assertIn('X-Requests-Remaining', resp.headers, 'failed verification should still show ratelimits') # verify once more that "left" is now at 0 self.assertEqual( resp.headers['X-Requests-Remaining'], HEADER_FORMAT.format(0, now + LIMIT_PER_TIME, 2), 'incorrect header format or values, most likely ' 'left is not 0 or resets changed since last request') # make sure I'm not banned lol #session.delete(API_ROOT + '/admin/bans/kenny2scratch') resp = session.post(API_ROOT + '/users/kenny2scratch/login') # this shouldn't 429 as it doesn't call Scratch self.assertNotEqual(resp.status_code, 429, 'login should not 429') # but it should include the header self.assertIn('X-Requests-Remaining', resp.headers, 'login missing custom header') # get this info for next test resets = re.search('resets=(\d+)', resp.headers['X-Requests-Remaining']) self.assertIsNotNone(resets, 'resets missing from header') resets = int(resets.group(1)) resp = session.post(API_ROOT + '/users/kenny2scratch/finish-login') now = int(http2time(resp.headers['Date'])) # this, however, should 429 self.assertEqual(resp.status_code, 429) self.assertIn('Retry-After', resp.headers, '429 missing Retry-After') self.assertEqual(int(resp.headers['Retry-After']), resets - now, 'wrong retry duration')
def _request(self, args): keep_alive_timeout = 0 url = args.url if not (url.lower().find('http://') == 0 or url.lower().find('https://') == 0): url = 'http://' + url default_port = {'http': 80, 'https': 443} r = urlparse.urlparse(url) isssl = r.scheme == 'https' and args.proxy == None path = r.path if not path: path = '/' if r.scheme not in default_port: raise CurlError(Curl.CURLE_UNSUPPORTED_PROTOCOL) if r.query: path = path + '?' + r.query port = r.port host = r.hostname if port is None: port = default_port[r.scheme] else: port = int(port) is_reuse = False target = '%s:%d' % (r.hostname, port) conn = None self._buf = '' if args.proxy: connecthost = args.proxy[0] connectport = args.proxy[1] else: connecthost = host connectport = port for i in range(2): if target not in self._conn_pool: conn = self._connect(connecthost, connectport, args.connect_timeout, isssl) else: keep_alive_timeout = self._timeout_pool[target] if keep_alive_timeout == 0 or time.time() <= keep_alive_timeout: conn = self._conn_pool[target] is_reuse = True else: continue del self._conn_pool[target] del self._timeout_pool[target] break if not conn: raise CurlError(Curl.CURLE_SEND_ERROR) conn.settimeout(20) self._conn = conn postdata = '' if args.raw: request, method = self._make_request(args.raw, url if args.proxy else path, host) else: method = None if args.request: method = args.request elif args.head: method = 'HEAD' elif args.upload_file: method = 'PUT' elif args.data: method = 'POST' else: method = 'GET' headers = {} if r.port: headers['Host'] = '%s:%d' % (r.hostname, port) else: headers['Host'] = r.hostname headers['User-Agent'] = args.user_agent if args.referer: headers['Referer'] = args.referer headers['Accept'] = '*/*' headers['Connection'] = 'Keep-Alive' if args.header: for line in args.header: pos = line.find(':') if pos > 0: key = line[:pos] val = line[pos + 1:].strip() for k in headers: if k.lower() == key.lower(): key = k break headers[key] = val if args.data: if len(args.data) == 1: postdata = args.data[0] else: for d in args.data: if postdata != '': postdata += '&' postdata += d headers['Content-Length'] = str(len(postdata)) if method == 'POST': if not headers.has_key('Content-Type'): headers['Content-Type'] = 'application/x-www-form-urlencoded' authinfo = None if args.user: authinfo = args.user if r.username: authinfo = r.username + ':' + r.password if authinfo: headers['Authorization'] = 'Basic ' + base64.b64encode(authinfo) cookie_str = str(self._init_cookie) if self._init_cookie else '' if target in self._cookie_pool: c = self._cookie_pool[target] for k in c.keys(): m = c[k] if r.path.find(m['path']) != 0: continue expires = m['expires'] if not expires: continue if cookiejar.http2time(expires) < time.time(): del c[k] cookie_str += c.output(attrs=[], header='', sep=';').strip() if args.cookie: if cookie_str: cookie_str += '; ' + args.cookie else: cookie_str = args.cookie if cookie_str: headers['Cookie'] = cookie_str if args.proxy: request = '%s %s HTTP/1.1\r\n' % (method, url) else: request = '%s %s HTTP/1.1\r\n' % (method, path) for k in headers: request += k + ': ' + headers[k] + '\r\n' request += '\r\n' response = '' content = '' msg = {} http_code = 0 mime_type = None for i in range(2): msg = {} response = '' mime_type = None try: if args.upload_file: conn.sendall(request) line = self._read_line() if line.find('100 Continue') != -1: self._read_line() conn.sendall(postdata) else: if response == '': cut = line.split() if len(cut) == 2: http_code = int(cut[1]) response += line elif postdata: conn.sendall(request + postdata) else: conn.sendall(request) except: raise CurlError(Curl.CURLE_SEND_ERROR) try: while True: line = self._read_line() if line == '\r\n' or line == '\n': response += line break elif line == '': raise CurlError(Curl.CURLE_RECV_ERROR) if response == '': cut = line.split() http_code = int(cut[1]) response += line pos = line.find(':') if pos == -1: continue end = line.find('\r') key = line[:pos].lower() val = line[pos + 1:end].strip() msg[key] = val if key == 'set-cookie': if target in self._cookie_pool: c = self._cookie_pool[target] else: c = cookies.SimpleCookie() self._cookie_pool[target] = c c.load(val) elif key == 'keep-alive': m = RE_KEEPALIVE_TIMEOUT.search(val) if m: keep_alive_timeout = int(m.group(1)) if keep_alive_timeout > 0: keep_alive_timeout += time.time() elif args.mime_type and key == 'content-type': m = RE_MIME_TYPE.search(val) if m: mime_type = m.group(1).strip() except CurlError as e: if i == 0 and is_reuse: conn = self._connect(host, port, args.connect_timeout, isssl) else: raise e else: break if args.mime_type and not (args.location and msg.has_key('location')): if not mime_type or mime_type.lower().find(args.mime_type.lower()) == -1: raise CurlError(Curl.CURLE_MIME_ERROR) if method != 'HEAD': if msg.get('transfer-encoding', None) == 'chunked': while True: chunk_size = int(self._read_line(), 16) if chunk_size > 0: content += self._read(chunk_size) self._read_line() if chunk_size <= 0: break else: content_len = msg.get('content-length', None) if content_len == None: if http_code != 204: content = self._read() elif content_len > 0: content_len = int(content_len) content = self._read(content_len) if len(content) != content_len: raise CurlError(Curl.CURLE_RECV_ERROR) encode = msg.get('content-encoding', None) if encode == 'gzip': content = gzip.GzipFile(fileobj=StringIO.StringIO(content)).read() elif encode == 'deflate': try: content = zlib.decompress(content, -zlib.MAX_WBITS) except: content = zlib.decompress(content) if msg.get('connection', '').find('close') != -1 or keep_alive_timeout > 0 and time.time() > keep_alive_timeout: conn.close() else: self._conn_pool[target] = conn self._timeout_pool[target] = keep_alive_timeout if msg.has_key('location') and args.location and (args.max_redirs == 0 or self._redirs < args.max_redirs): self._redirs += 1 args.data = '' location_url = '' if msg['location'].startswith('http'): location_url = msg['location'] elif msg['location'].startswith('/'): location_url = '%s://%s%s' % (r.scheme, r.netloc, msg['location']) if args.url != location_url: args.url = location_url return self._request(args) self._error_count = 0 if self.sniff_func: self.sniff_func(args.url, response, content) return (http_code, response, content, 0, url)