def parse_ns_headers(ns_headers):
"""Ad-hoc parser for Netscape protocol cookie-attributes.
The old Netscape cookie format for Set-Cookie can for instance contain
an unquoted "," in the expires field, so we have to use this ad-hoc
parser instead of split_header_words.
XXX This may not make the best possible effort to parse all the crap
that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient
parser is probably better, so could do worse than following that if
this ever gives any trouble.
Currently, this is also used for parsing RFC 2109 cookies.
"""
known_attrs = (
"expires",
"domain",
"path",
"secure",
# RFC 2109 attrs (may turn up in Netscape cookies, too)
"version",
"port",
"max-age")
result = []
for ns_header in ns_headers:
pairs = []
version_set = False
params = re.split(r";\s*", ns_header)
for ii in range(len(params)):
param = params[ii]
param = param.rstrip()
if param == "": continue
if "=" not in param:
k, v = param, None
else:
k, v = re.split(r"\s*=\s*", param, 1)
k = k.lstrip()
if ii != 0:
lc = k.lower()
if lc in known_attrs:
k = lc
if k == "version":
# This is an RFC 2109 cookie.
v = strip_quotes(v)
version_set = True
if k == "expires":
# convert expires date to seconds since epoch
v = http2time(strip_quotes(v)) # None if invalid
pairs.append((k, v))
if pairs:
if not version_set:
pairs.append(("version", "0"))
result.append(pairs)
return result
def test_set_cookie():
cookiejar = FileCookieJar()
_cookie = {"value_0": "v_0", "value_1": "v_1", "value_2": "v_2"}
c = SimpleCookie(_cookie)
domain_0 = ".test_domain"
domain_1 = "test_domain"
max_age = "09 Feb 1994 22:23:32 GMT"
expires = http2time(max_age)
path = "test/path"
c["value_0"]["max-age"] = max_age
c["value_0"]["domain"] = domain_0
c["value_0"]["path"] = path
c["value_1"]["domain"] = domain_1
util.set_cookie(cookiejar, c)
cookies = cookiejar._cookies
c_0 = cookies[domain_0][path]["value_0"]
c_1 = cookies[domain_1][""]["value_1"]
c_2 = cookies[""][""]["value_2"]
assert not (c_2.domain_specified and c_2.path_specified)
assert c_1.domain_specified and not c_1.domain_initial_dot and not \
c_1.path_specified
assert c_0.domain_specified and c_0.domain_initial_dot and \
c_0.path_specified
assert c_0.expires == expires
assert c_0.domain == domain_0
assert c_0.name == "value_0"
assert c_0.path == path
assert c_0.value == "v_0"
assert not c_1.expires
assert c_1.domain == domain_1
assert c_1.name == "value_1"
assert c_1.path == ""
assert c_1.value == "v_1"
assert not c_2.expires
assert c_2.domain == ""
assert c_2.name == "value_2"
assert c_2.path == ""
assert c_2.value == "v_2"
def test_set_cookie():
cookiejar = FileCookieJar()
_cookie = {"value_0": "v_0", "value_1": "v_1", "value_2": "v_2"}
c = SimpleCookie(_cookie)
domain_0 = ".test_domain"
domain_1 = "test_domain"
max_age = "09 Feb 1994 22:23:32 GMT"
expires = http2time(max_age)
path = "test/path"
c["value_0"]["max-age"] = max_age
c["value_0"]["domain"] = domain_0
c["value_0"]["path"] = path
c["value_1"]["domain"] = domain_1
util.set_cookie(cookiejar, c)
cookies = cookiejar._cookies
c_0 = cookies[domain_0][path]["value_0"]
c_1 = cookies[domain_1][""]["value_1"]
c_2 = cookies[""][""]["value_2"]
assert not (c_2.domain_specified and c_2.path_specified)
assert c_1.domain_specified and not c_1.domain_initial_dot and not c_1.path_specified
assert c_0.domain_specified and c_0.domain_initial_dot and c_0.path_specified
assert c_0.expires == expires
assert c_0.domain == domain_0
assert c_0.name == "value_0"
assert c_0.path == path
assert c_0.value == "v_0"
assert not c_1.expires
assert c_1.domain == domain_1
assert c_1.name == "value_1"
assert c_1.path == ""
assert c_1.value == "v_1"
assert not c_2.expires
assert c_2.domain == ""
assert c_2.name == "value_2"
assert c_2.path == ""
assert c_2.value == "v_2"
def make(self, cookie_string):
# split "Set-Cookie:x=y; domain=...; expires=...;..."
set_string, tuple_string = cookie_string.split(":", 1)
# parse version from set string
version = self._version(set_string)
# parse name, value from tuple string
nv = self._name_value(tuple_string)
# change tuple string to dict
dict = self._dict(tuple_string)
if nv is None or dict is None:
return None
name, value = nv
port = dict.get("port", None)
port_specified = port is not None
domain = dict.get("domain", None)
domain_specified = domain is not None
domain_initial_dot = False
if domain is not None:
domain_initial_dot = domain.startswith(".")
path = dict.get("path", None)
path_specified = path is not None
secure = dict.get("secure", False)
expires = dict.get("expires", None)
if expires is not None:
expires = http2time(expires)
discard = dict.get("discard", False)
comment = dict.get("comment", None)
comment_url = None
rest = {}
# create cookielib.Cookie object
cookie = Cookie(version, name, value, port, port_specified, domain,
domain_specified, domain_initial_dot, path,
path_specified, secure, expires, discard, comment,
comment_url, rest)
return cookie
def set_cookie(cookiejar, kaka):
"""PLaces a cookie (a cookielib.Cookie based on a set-cookie header
line) in the cookie jar.
Always chose the shortest expires time.
:param cookiejar:
:param kaka: Cookie
"""
# default rfc2109=False
# max-age, httponly
for cookie_name, morsel in kaka.items():
std_attr = ATTRS.copy()
std_attr["name"] = cookie_name
_tmp = morsel.coded_value
if _tmp.startswith('"') and _tmp.endswith('"'):
std_attr["value"] = _tmp[1:-1]
else:
std_attr["value"] = _tmp
std_attr["version"] = 0
attr = ""
# copy attributes that have values
try:
for attr in morsel.keys():
if attr in ATTRS:
if morsel[attr]:
if attr == "expires":
std_attr[attr] = http2time(morsel[attr])
else:
std_attr[attr] = morsel[attr]
elif attr == "max-age":
if morsel[attr]:
std_attr["expires"] = http2time(morsel[attr])
except TimeFormatError:
# Ignore cookie
logger.info(
"Time format error on %s parameter in received cookie" %
(sanitize(attr), ))
continue
for att, spec in PAIRS.items():
if std_attr[att]:
std_attr[spec] = True
if std_attr["domain"] and std_attr["domain"].startswith("."):
std_attr["domain_initial_dot"] = True
if morsel["max-age"] is 0:
try:
cookiejar.clear(domain=std_attr["domain"],
path=std_attr["path"],
name=std_attr["name"])
except ValueError:
pass
else:
# Fix for Microsoft cookie error
if "version" in std_attr:
try:
std_attr["version"] = std_attr["version"].split(",")[0]
except (TypeError, AttributeError):
pass
new_cookie = Cookie(**std_attr)
cookiejar.set_cookie(new_cookie)
def set_cookie(cookiejar, kaka):
"""PLaces a cookie (a http_cookielib.Cookie based on a set-cookie header
line) in the cookie jar.
Always chose the shortest expires time.
:param cookiejar:
:param kaka: Cookie
"""
# default rfc2109=False
# max-age, httponly
for cookie_name, morsel in kaka.items():
std_attr = ATTRS.copy()
std_attr["name"] = cookie_name
_tmp = morsel.coded_value
if _tmp.startswith('"') and _tmp.endswith('"'):
std_attr["value"] = _tmp[1:-1]
else:
std_attr["value"] = _tmp
std_attr["version"] = 0
attr = ""
# copy attributes that have values
try:
for attr in morsel.keys():
if attr in ATTRS:
if morsel[attr]:
if attr == "expires":
std_attr[attr] = http2time(morsel[attr])
else:
std_attr[attr] = morsel[attr]
elif attr == "max-age":
if morsel[attr]:
std_attr["expires"] = http2time(morsel[attr])
except TimeFormatError:
# Ignore cookie
logger.info(
"Time format error on %s parameter in received cookie" % (
sanitize(attr),))
continue
for att, spec in PAIRS.items():
if std_attr[att]:
std_attr[spec] = True
if std_attr["domain"] and std_attr["domain"].startswith("."):
std_attr["domain_initial_dot"] = True
if morsel["max-age"] == 0:
try:
cookiejar.clear(domain=std_attr["domain"],
path=std_attr["path"],
name=std_attr["name"])
except ValueError:
pass
else:
# Fix for Microsoft cookie error
if "version" in std_attr:
try:
std_attr["version"] = std_attr["version"].split(",")[0]
except (TypeError, AttributeError):
pass
new_cookie = http_cookiejar.Cookie(**std_attr)
cookiejar.set_cookie(new_cookie)
def _http(self,
url,
post=None,
headers={},
method=None,
proxy=None,
cookcookie=True,
location=True,
locationcount=0):
if not method:
if post:
method = "POST"
else:
method = "GET"
rep = None
urlinfo = https, host, port, path = self._get_urlinfo(url)
log = {}
con = self.conpool._get_connect(urlinfo, proxy)
# con .set_debuglevel(2) #?
conerr = False
try:
con._send_output = self._send_output(con._send_output, con, log)
tmpheaders = copy.deepcopy(headers)
tmpheaders['Accept-Encoding'] = 'gzip, deflate'
tmpheaders['Connection'] = 'Keep-Alive'
tmpheaders[
'User-Agent'] = tmpheaders['User-Agent'] if tmpheaders.get(
'User-Agent'
) else 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36'
if cookcookie:
c = self.cookiepool.get(host, None)
if not c:
self.cookiepool[host] = self.initcookie
c = self.cookiepool.get(host)
if 'Cookie' in tmpheaders:
cookie_str = tmpheaders['Cookie'].strip()
if not cookie_str.endswith(';'):
cookie_str += ";"
for cookiepart in cookie_str.split(";"):
if cookiepart.strip() != "":
cookiekey, cookievalue = cookiepart.split("=", 1)
c[cookiekey.strip()] = cookievalue.strip()
for k in c.keys():
m = c[k]
# check cookie path
if path.find(m['path']) != 0:
continue
expires = m['expires']
if not expires:
continue
# check cookie expires time
if cookiejar.http2time(expires) < time.time():
del c[k]
cookie_str = c.output(attrs=[], header='', sep=';').strip()
if cookie_str:
tmpheaders['Cookie'] = cookie_str
if post:
tmpheaders['Content-Type'] = tmpheaders.get(
'Content-Type', 'application/x-www-form-urlencoded')
else:
# content-length err 411
tmpheaders['Content-Length'] = tmpheaders.get(
'Content-Length', 0)
if method == 'GET':
del tmpheaders['Content-Length']
con.request(method, path, post, tmpheaders)
rep = con.getresponse()
body = rep.read()
encode = rep.msg.get('content-encoding', None)
if encode == 'gzip':
body = gzip.GzipFile(fileobj=StringIO.StringIO(body)).read()
elif encode == 'deflate':
try:
body = zlib.decompress(body, -zlib.MAX_WBITS)
except:
body = zlib.decompress(body)
body = self._decode_html(rep.msg.dict.get('content-type', ''),
body)
retheader = Compatibleheader(str(rep.msg))
retheader.setdict(rep.msg.dict)
redirect = rep.msg.dict.get('location', url)
if not redirect.startswith('http'):
redirect = urlparse.urljoin(url, redirect)
if cookcookie and "set-cookie" in rep.msg.dict:
c = self.cookiepool[host]
c.load(rep.msg.dict['set-cookie'])
except http.client.ImproperConnectionState:
conerr = True
raise
except:
raise
finally:
if conerr or (rep
and rep.msg.get('connection') == 'close') or proxy:
self.conpool._release_connect(urlinfo)
con.close()
else:
self.conpool._put_connect(urlinfo, con)
log["url"] = url
if post:
log['request'] += "\r\n\r\n" + post
log["response"] = "HTTP/%.1f %d %s" % (
rep.version * 0.1, rep.status,
rep.reason) + '\r\n' + str(retheader) + '\r\n' + (body[:4096])
if location and url != redirect and locationcount < 5:
method = 'HEAD' if method == 'HEAD' else 'GET'
a, b, c, d, e = self._http(redirect,
method=method,
proxy=proxy,
cookcookie=cookcookie,
location=location,
locationcount=locationcount + 1)
log["response"] = e["response"]
return a, b, c, d, log
return rep.status, retheader, body, redirect, log
def test_ratelimiting(self):
"""Test actually being ratelimited."""
# bypass auth and make all verifications successful
session.post(API_ROOT + '/debug/3')
set_session(0)
# set up a ratelimit that is easily testable
session.post(API_ROOT + '/admin/ratelimits/kenny2scratch',
json={'ratelimit': 2})
resp = session.put(API_ROOT + '/verify/deathly_hallows')
now = int(http2time(resp.headers['Date']))
# anything that calls the verify API returns a X-Requests-Remaining
# header like left=2, resets=1598514504, to=2
self.assertIn('X-Requests-Remaining', resp.headers, 'no custom header')
# *starting* verification should not trigger ratelimits
self.assertEqual(resp.headers['X-Requests-Remaining'],
HEADER_FORMAT.format(2, now + LIMIT_PER_TIME, 2),
'incorrect header format or values')
time.sleep(2)
# the reset time should always be LIMIT_PER_TIME seconds into the future
# up until "left" is less than "to"
resp = session.post(API_ROOT + '/verify/deathly_hallows')
now = int(http2time(resp.headers['Date']))
# the POST API should return it too
self.assertIn('X-Requests-Remaining', resp.headers, 'no custom header')
# *finishing* verification should decrement "left"
self.assertEqual(
resp.headers['X-Requests-Remaining'],
HEADER_FORMAT.format(1, now + LIMIT_PER_TIME, 2),
'incorrect header format or values, most likely '
'left is not 1')
time.sleep(2)
resp = session.put(API_ROOT + '/verify/deathly_hallows')
# now that left < to, resets should still be LIMIT_PER_TIME after the
# "now" that was *before* the last POST, rather than LIMIT_PER_TIME
# after *this* "now". Also, this shouldn't decrement "left"
self.assertEqual(
resp.headers['X-Requests-Remaining'],
HEADER_FORMAT.format(1, now + LIMIT_PER_TIME, 2),
'incorrect header format or values, most likely '
'left is not 1 or resets changed since last request')
# don't pretend it worked
session.post(API_ROOT + '/debug/1')
resp = session.post(API_ROOT + '/verify/deathly_hallows')
self.assertEqual(resp.status_code, 403, "wait, didn't fail?")
# even on failed verification, the API was called
self.assertIn('X-Requests-Remaining', resp.headers,
'failed verification should still show ratelimits')
# verify once more that "left" is now at 0
self.assertEqual(
resp.headers['X-Requests-Remaining'],
HEADER_FORMAT.format(0, now + LIMIT_PER_TIME, 2),
'incorrect header format or values, most likely '
'left is not 0 or resets changed since last request')
# make sure I'm not banned lol
#session.delete(API_ROOT + '/admin/bans/kenny2scratch')
resp = session.post(API_ROOT + '/users/kenny2scratch/login')
# this shouldn't 429 as it doesn't call Scratch
self.assertNotEqual(resp.status_code, 429, 'login should not 429')
# but it should include the header
self.assertIn('X-Requests-Remaining', resp.headers,
'login missing custom header')
# get this info for next test
resets = re.search('resets=(\d+)',
resp.headers['X-Requests-Remaining'])
self.assertIsNotNone(resets, 'resets missing from header')
resets = int(resets.group(1))
resp = session.post(API_ROOT + '/users/kenny2scratch/finish-login')
now = int(http2time(resp.headers['Date']))
# this, however, should 429
self.assertEqual(resp.status_code, 429)
self.assertIn('Retry-After', resp.headers, '429 missing Retry-After')
self.assertEqual(int(resp.headers['Retry-After']), resets - now,
'wrong retry duration')
def _request(self, args):
keep_alive_timeout = 0
url = args.url
if not (url.lower().find('http://') == 0 or url.lower().find('https://') == 0):
url = 'http://' + url
default_port = {'http': 80, 'https': 443}
r = urlparse.urlparse(url)
isssl = r.scheme == 'https' and args.proxy == None
path = r.path
if not path:
path = '/'
if r.scheme not in default_port:
raise CurlError(Curl.CURLE_UNSUPPORTED_PROTOCOL)
if r.query:
path = path + '?' + r.query
port = r.port
host = r.hostname
if port is None:
port = default_port[r.scheme]
else:
port = int(port)
is_reuse = False
target = '%s:%d' % (r.hostname, port)
conn = None
self._buf = ''
if args.proxy:
connecthost = args.proxy[0]
connectport = args.proxy[1]
else:
connecthost = host
connectport = port
for i in range(2):
if target not in self._conn_pool:
conn = self._connect(connecthost, connectport, args.connect_timeout, isssl)
else:
keep_alive_timeout = self._timeout_pool[target]
if keep_alive_timeout == 0 or time.time() <= keep_alive_timeout:
conn = self._conn_pool[target]
is_reuse = True
else:
continue
del self._conn_pool[target]
del self._timeout_pool[target]
break
if not conn:
raise CurlError(Curl.CURLE_SEND_ERROR)
conn.settimeout(20)
self._conn = conn
postdata = ''
if args.raw:
request, method = self._make_request(args.raw, url if args.proxy else path, host)
else:
method = None
if args.request:
method = args.request
elif args.head:
method = 'HEAD'
elif args.upload_file:
method = 'PUT'
elif args.data:
method = 'POST'
else:
method = 'GET'
headers = {}
if r.port:
headers['Host'] = '%s:%d' % (r.hostname, port)
else:
headers['Host'] = r.hostname
headers['User-Agent'] = args.user_agent
if args.referer:
headers['Referer'] = args.referer
headers['Accept'] = '*/*'
headers['Connection'] = 'Keep-Alive'
if args.header:
for line in args.header:
pos = line.find(':')
if pos > 0:
key = line[:pos]
val = line[pos + 1:].strip()
for k in headers:
if k.lower() == key.lower():
key = k
break
headers[key] = val
if args.data:
if len(args.data) == 1:
postdata = args.data[0]
else:
for d in args.data:
if postdata != '':
postdata += '&'
postdata += d
headers['Content-Length'] = str(len(postdata))
if method == 'POST':
if not headers.has_key('Content-Type'):
headers['Content-Type'] = 'application/x-www-form-urlencoded'
authinfo = None
if args.user:
authinfo = args.user
if r.username:
authinfo = r.username + ':' + r.password
if authinfo:
headers['Authorization'] = 'Basic ' + base64.b64encode(authinfo)
cookie_str = str(self._init_cookie) if self._init_cookie else ''
if target in self._cookie_pool:
c = self._cookie_pool[target]
for k in c.keys():
m = c[k]
if r.path.find(m['path']) != 0:
continue
expires = m['expires']
if not expires:
continue
if cookiejar.http2time(expires) < time.time():
del c[k]
cookie_str += c.output(attrs=[], header='', sep=';').strip()
if args.cookie:
if cookie_str:
cookie_str += '; ' + args.cookie
else:
cookie_str = args.cookie
if cookie_str:
headers['Cookie'] = cookie_str
if args.proxy:
request = '%s %s HTTP/1.1\r\n' % (method, url)
else:
request = '%s %s HTTP/1.1\r\n' % (method, path)
for k in headers:
request += k + ': ' + headers[k] + '\r\n'
request += '\r\n'
response = ''
content = ''
msg = {}
http_code = 0
mime_type = None
for i in range(2):
msg = {}
response = ''
mime_type = None
try:
if args.upload_file:
conn.sendall(request)
line = self._read_line()
if line.find('100 Continue') != -1:
self._read_line()
conn.sendall(postdata)
else:
if response == '':
cut = line.split()
if len(cut) == 2:
http_code = int(cut[1])
response += line
elif postdata:
conn.sendall(request + postdata)
else:
conn.sendall(request)
except:
raise CurlError(Curl.CURLE_SEND_ERROR)
try:
while True:
line = self._read_line()
if line == '\r\n' or line == '\n':
response += line
break
elif line == '':
raise CurlError(Curl.CURLE_RECV_ERROR)
if response == '':
cut = line.split()
http_code = int(cut[1])
response += line
pos = line.find(':')
if pos == -1:
continue
end = line.find('\r')
key = line[:pos].lower()
val = line[pos + 1:end].strip()
msg[key] = val
if key == 'set-cookie':
if target in self._cookie_pool:
c = self._cookie_pool[target]
else:
c = cookies.SimpleCookie()
self._cookie_pool[target] = c
c.load(val)
elif key == 'keep-alive':
m = RE_KEEPALIVE_TIMEOUT.search(val)
if m:
keep_alive_timeout = int(m.group(1))
if keep_alive_timeout > 0:
keep_alive_timeout += time.time()
elif args.mime_type and key == 'content-type':
m = RE_MIME_TYPE.search(val)
if m:
mime_type = m.group(1).strip()
except CurlError as e:
if i == 0 and is_reuse:
conn = self._connect(host, port, args.connect_timeout, isssl)
else:
raise e
else:
break
if args.mime_type and not (args.location and msg.has_key('location')):
if not mime_type or mime_type.lower().find(args.mime_type.lower()) == -1:
raise CurlError(Curl.CURLE_MIME_ERROR)
if method != 'HEAD':
if msg.get('transfer-encoding', None) == 'chunked':
while True:
chunk_size = int(self._read_line(), 16)
if chunk_size > 0:
content += self._read(chunk_size)
self._read_line()
if chunk_size <= 0:
break
else:
content_len = msg.get('content-length', None)
if content_len == None:
if http_code != 204:
content = self._read()
elif content_len > 0:
content_len = int(content_len)
content = self._read(content_len)
if len(content) != content_len:
raise CurlError(Curl.CURLE_RECV_ERROR)
encode = msg.get('content-encoding', None)
if encode == 'gzip':
content = gzip.GzipFile(fileobj=StringIO.StringIO(content)).read()
elif encode == 'deflate':
try:
content = zlib.decompress(content, -zlib.MAX_WBITS)
except:
content = zlib.decompress(content)
if msg.get('connection', '').find('close') != -1 or keep_alive_timeout > 0 and time.time() > keep_alive_timeout:
conn.close()
else:
self._conn_pool[target] = conn
self._timeout_pool[target] = keep_alive_timeout
if msg.has_key('location') and args.location and (args.max_redirs == 0 or self._redirs < args.max_redirs):
self._redirs += 1
args.data = ''
location_url = ''
if msg['location'].startswith('http'):
location_url = msg['location']
elif msg['location'].startswith('/'):
location_url = '%s://%s%s' % (r.scheme, r.netloc, msg['location'])
if args.url != location_url:
args.url = location_url
return self._request(args)
self._error_count = 0
if self.sniff_func:
self.sniff_func(args.url, response, content)
return (http_code,
response,
content,
0,
url)
