def test_processors(self):
# *_request / *_response methods get called appropriately
o = OpenerDirector()
meth_spec = [
[("http_request", "return request"),
("http_response", "return response")],
[("http_request", "return request"),
("http_response", "return response")],
]
handlers = add_ordered_mock_handlers(o, meth_spec)
req = Request("http://example.com/")
r = o.open(req)
# processor methods are called on *all* handlers that define them,
# not just the first handler that handles the request
calls = [(handlers[0], "http_request"), (handlers[1], "http_request"),
(handlers[0], "http_response"),
(handlers[1], "http_response")]
for i, (handler, name, args, kwds) in enumerate(o.calls):
if i < 2:
# *_request
self.assertEqual((handler, name), calls[i])
self.assertEqual(len(args), 1)
self.assert_(isinstance(args[0], Request))
else:
# *_response
self.assertEqual((handler, name), calls[i])
self.assertEqual(len(args), 2)
self.assert_(isinstance(args[0], Request))
# response from opener.open is None, because there's no
# handler that defines http_open to handle it
self.assert_(args[1] is None
or isinstance(args[1], MockResponse))
def test_handled(self):
# handler returning non-None means no more handlers will be called
o = OpenerDirector()
meth_spec = [
["http_open", "ftp_open", "http_error_302"],
["ftp_open"],
[("http_open", "return self")],
[("http_open", "return self")],
]
handlers = add_ordered_mock_handlers(o, meth_spec)
req = Request("http://example.com/")
r = o.open(req)
# Second .http_open() gets called, third doesn't, since second returned
# non-None. Handlers without .http_open() never get any methods called
# on them.
# In fact, second mock handler defining .http_open() returns self
# (instead of response), which becomes the OpenerDirector's return
# value.
self.assertEqual(r, handlers[2])
calls = [(handlers[0], "http_open"), (handlers[2], "http_open")]
for expected, got in zip(calls, o.calls):
handler, name, args, kwds = got
self.assertEqual((handler, name), expected)
self.assertEqual(args, (req, ))
def test_http_error(self):
# XXX http_error_default
# http errors are a special case
o = OpenerDirector()
meth_spec = [
[("http_open", "error 302")],
[("http_error_400", "raise"), "http_open"],
[("http_error_302", "return response"), "http_error_303",
"http_error"],
[("http_error_302")],
]
handlers = add_ordered_mock_handlers(o, meth_spec)
class Unknown:
def __eq__(self, other): return True
req = Request("http://example.com/")
r = o.open(req)
assert len(o.calls) == 2
calls = [(handlers[0], "http_open", (req,)),
(handlers[2], "http_error_302",
(req, Unknown(), 302, "", {}))]
for expected, got in zip(calls, o.calls):
handler, method_name, args = expected
self.assertEqual((handler, method_name), got[:2])
self.assertEqual(args, got[2])
def test_processors(self):
# *_request / *_response methods get called appropriately
o = OpenerDirector()
meth_spec = [
[("http_request", "return request"),
("http_response", "return response")],
[("http_request", "return request"),
("http_response", "return response")],
]
handlers = add_ordered_mock_handlers(o, meth_spec)
req = Request("http://example.com/")
r = o.open(req)
# processor methods are called on *all* handlers that define them,
# not just the first handler that handles the request
calls = [
(handlers[0], "http_request"), (handlers[1], "http_request"),
(handlers[0], "http_response"), (handlers[1], "http_response")]
for i, (handler, name, args, kwds) in enumerate(o.calls):
if i < 2:
# *_request
self.assertEqual((handler, name), calls[i])
self.assertEqual(len(args), 1)
self.assertIsInstance(args[0], Request)
else:
# *_response
self.assertEqual((handler, name), calls[i])
self.assertEqual(len(args), 2)
self.assertIsInstance(args[0], Request)
# response from opener.open is None, because there's no
# handler that defines http_open to handle it
self.assertTrue(args[1] is None or
isinstance(args[1], MockResponse))
def test_handled(self):
# handler returning non-None means no more handlers will be called
o = OpenerDirector()
meth_spec = [
["http_open", "ftp_open", "http_error_302"],
["ftp_open"],
[("http_open", "return self")],
[("http_open", "return self")],
]
handlers = add_ordered_mock_handlers(o, meth_spec)
req = Request("http://example.com/")
r = o.open(req)
# Second .http_open() gets called, third doesn't, since second returned
# non-None. Handlers without .http_open() never get any methods called
# on them.
# In fact, second mock handler defining .http_open() returns self
# (instead of response), which becomes the OpenerDirector's return
# value.
self.assertEqual(r, handlers[2])
calls = [(handlers[0], "http_open"), (handlers[2], "http_open")]
for expected, got in zip(calls, o.calls):
handler, name, args, kwds = got
self.assertEqual((handler, name), expected)
self.assertEqual(args, (req,))
def test_http_error(self):
# XXX http_error_default
# http errors are a special case
o = OpenerDirector()
meth_spec = [
[("http_open", "error 302")],
[("http_error_400", "raise"), "http_open"],
[("http_error_302", "return response"), "http_error_303",
"http_error"],
[("http_error_302")],
]
handlers = add_ordered_mock_handlers(o, meth_spec)
class Unknown:
def __eq__(self, other):
return True
req = Request("http://example.com/")
r = o.open(req)
assert len(o.calls) == 2
calls = [(handlers[0], "http_open", (req, )),
(handlers[2], "http_error_302", (req, Unknown(), 302, "", {}))
]
for expected, got in zip(calls, o.calls):
handler, method_name, args = expected
self.assertEqual((handler, method_name), got[:2])
self.assertEqual(args, got[2])
def test_proxy_https(self):
o = OpenerDirector()
ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128"))
o.add_handler(ph)
meth_spec = [[("https_open", "return response")]]
handlers = add_ordered_mock_handlers(o, meth_spec)
req = Request("https://www.example.com/")
self.assertEqual(req.get_host(), "www.example.com")
r = o.open(req)
self.assertEqual(req.get_host(), "proxy.example.com:3128")
self.assertEqual([(handlers[0], "https_open")], [tup[0:2] for tup in o.calls])
def test_proxy_https(self):
o = OpenerDirector()
ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128"))
o.add_handler(ph)
meth_spec = [[("https_open", "return response")]]
handlers = add_ordered_mock_handlers(o, meth_spec)
req = Request("https://www.example.com/")
self.assertEqual(req.get_host(), "www.example.com")
r = o.open(req)
self.assertEqual(req.get_host(), "proxy.example.com:3128")
self.assertEqual([(handlers[0], "https_open")],
[tup[0:2] for tup in o.calls])
def post_info(url, **kwargs):
# opener = build_opener()
opener = OpenerDirector()
opener.add_handler(HTTPHandler())
form_data = urlencode(kwargs)
req = Request(
url,
data=form_data,
headers={
'Content-Type': 'application/x-www-form-urlencoded',
}
)
return opener.open(req)
def close(self):
OpenerDirector.close(self)
# make it very obvious this object is no longer supposed to be used
self.open = self.error = self.retrieve = self.add_handler = None
if self._tempfiles:
for filename in self._tempfiles:
try:
os.unlink(filename)
except OSError:
pass
del self._tempfiles[:]
def test_proxy_no_proxy(self):
os.environ['no_proxy'] = 'python.org'
o = OpenerDirector()
ph = urllib.request.ProxyHandler(dict(http="proxy.example.com"))
o.add_handler(ph)
req = Request("http://www.perl.org/")
self.assertEqual(req.get_host(), "www.perl.org")
r = o.open(req)
self.assertEqual(req.get_host(), "proxy.example.com")
req = Request("http://www.python.org")
self.assertEqual(req.get_host(), "www.python.org")
r = o.open(req)
self.assertEqual(req.get_host(), "www.python.org")
del os.environ['no_proxy']
def test_basic_auth(self, quote_char='"'):
opener = OpenerDirector()
password_manager = MockPasswordManager()
auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
realm = "ACME Widget Store"
http_handler = MockHTTPHandler(
401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' %
(quote_char, realm, quote_char) )
opener.add_handler(auth_handler)
opener.add_handler(http_handler)
self._test_basic_auth(opener, auth_handler, "Authorization",
realm, http_handler, password_manager,
"http://acme.example.com/protected",
"http://acme.example.com/protected",
)
def add_handler(self, handler):
if not isinstance(handler, BaseHandler):
raise Exception("%r have to be a %r instance" %
(handler, BaseHandler))
if not isinstance(EndPoint.opener, OpenerDirector):
EndPoint.opener = OpenerDirector()
EndPoint.opener.add_handler(handler)
def test_proxy_basic_auth(self):
opener = OpenerDirector()
ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128"))
opener.add_handler(ph)
password_manager = MockPasswordManager()
auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager)
realm = "ACME Networks"
http_handler = MockHTTPHandler(
407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm)
opener.add_handler(auth_handler)
opener.add_handler(http_handler)
self._test_basic_auth(opener, auth_handler, "Proxy-authorization",
realm, http_handler, password_manager,
"http://acme.example.com:3128/protected",
"proxy.example.com:3128",
)
def test_badly_named_methods(self):
# test work-around for three methods that accidentally follow the
# naming conventions for handler methods
# (*_open() / *_request() / *_response())
# These used to call the accidentally-named methods, causing a
# TypeError in real code; here, returning self from these mock
# methods would either cause no exception, or AttributeError.
from urllib.error import URLError
o = OpenerDirector()
meth_spec = [[("do_open", "return self"), ("proxy_open", "return self")], [("redirect_request", "return self")]]
handlers = add_ordered_mock_handlers(o, meth_spec)
o.add_handler(urllib.request.UnknownHandler())
for scheme in "do", "proxy", "redirect":
self.assertRaises(URLError, o.open, scheme + "://example.com/")
def test_handler_order(self):
o = OpenerDirector()
handlers = []
for meths, handler_order in [
([("http_open", "return self")], 500),
(["http_open"], 0),
]:
class MockHandlerSubclass(MockHandler): pass
h = MockHandlerSubclass(meths)
h.handler_order = handler_order
handlers.append(h)
o.add_handler(h)
r = o.open("http://example.com/")
# handlers called in reverse order, thanks to their sort order
self.assertEqual(o.calls[0][0], handlers[1])
self.assertEqual(o.calls[1][0], handlers[0])
def build_tor_opener(guard, hops_count=3, debuglevel=0):
opener = OpenerDirector()
default_classes = [
ProxyHandler, UnknownHandler, HTTPDefaultErrorHandler,
HTTPRedirectHandler, HTTPErrorProcessor
]
for cls in default_classes:
opener.add_handler(cls())
opener.add_handler(TorHTTPHandler(guard, hops_count,
debuglevel=debuglevel))
opener.add_handler(
TorHTTPSHandler(guard, hops_count, debuglevel=debuglevel))
opener.addheaders = []
return opener
def __init__(self, elem=None, parent=None, method=lambda: None):
self.elem = elem
self.parent = parent
self.method = method
# initialize opener only once
if EndPoint.opener is None:
EndPoint.opener = OpenerDirector()
EndPoint.opener.add_handler(HTTPHandler())
EndPoint.opener.add_handler(HTTPSHandler(context=CTX))
def save_to_file(url: str, file_path: str, opener: OpenerDirector = None):
def process(request_):
with open(file_path, 'wb') as f:
f.write(request_.read())
if opener:
with closing(opener.open(url)) as request:
process(request)
else:
with closing(urlopen(url)) as request:
process(request)
class TestClient(object):
def __init__(self, host, port, timeout=5):
self.host = host
self.port = port
self.timeout = timeout
opener_director = OpenerDirector()
opener_director.add_handler(HTTPHandler())
def send_request(self, url, headers={}, arguments=None):
if not isinstance(arguments, bytes):
arguments = bytes(json.dumps(arguments), 'ascii')
request = Request(method='POST',
url=url,
headers=headers,
data=arguments)
with self.opener_director.open(request,
timeout=self.timeout) as response:
if response.status != 200:
raise HTTPError(url, response.status, response.msg,
response.headers, None)
return json.loads(str(response.read(), 'ascii'))
class Scope(object):
def __init__(self, harness):
self.harness = harness
self.id = uuid.uuid4().hex
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
pass
def __repr__(self):
return '{}({})'.format(type(self).__name__, repr(self.id))
def send_request(self, arguments=None):
return self.harness.send_request(url='http://{}:{}/{}'.format(
self.harness.host, self.harness.port, self.id),
headers={
'Accept': 'application/json',
'Content-Type':
'application/json'
},
arguments=arguments)
def url(self, path=''):
return 'http://{}:{}/{}.{}'.format(self.harness.host,
self.harness.port, self.id,
path)
def scope(self):
return TestClient.Scope(self)
def test_badly_named_methods(self):
# test work-around for three methods that accidentally follow the
# naming conventions for handler methods
# (*_open() / *_request() / *_response())
# These used to call the accidentally-named methods, causing a
# TypeError in real code; here, returning self from these mock
# methods would either cause no exception, or AttributeError.
from urllib.error import URLError
o = OpenerDirector()
meth_spec = [
[("do_open", "return self"), ("proxy_open", "return self")],
[("redirect_request", "return self")],
]
handlers = add_ordered_mock_handlers(o, meth_spec)
o.add_handler(urllib.request.UnknownHandler())
for scheme in "do", "proxy", "redirect":
self.assertRaises(URLError, o.open, scheme + "://example.com/")
def test_handler_order(self):
o = OpenerDirector()
handlers = []
for meths, handler_order in [
([("http_open", "return self")], 500),
(["http_open"], 0),
]:
class MockHandlerSubclass(MockHandler):
pass
h = MockHandlerSubclass(meths)
h.handler_order = handler_order
handlers.append(h)
o.add_handler(h)
r = o.open("http://example.com/")
# handlers called in reverse order, thanks to their sort order
self.assertEqual(o.calls[0][0], handlers[1])
self.assertEqual(o.calls[1][0], handlers[0])
def test_basic_auth(self, quote_char='"'):
opener = OpenerDirector()
password_manager = MockPasswordManager()
auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
realm = "ACME Widget Store"
http_handler = MockHTTPHandler(
401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' %
(quote_char, realm, quote_char))
opener.add_handler(auth_handler)
opener.add_handler(http_handler)
self._test_basic_auth(
opener,
auth_handler,
"Authorization",
realm,
http_handler,
password_manager,
"http://acme.example.com/protected",
"http://acme.example.com/protected",
)
def test_raise(self):
# raising URLError stops processing of request
o = OpenerDirector()
meth_spec = [
[("http_open", "raise")],
[("http_open", "return self")],
]
handlers = add_ordered_mock_handlers(o, meth_spec)
req = Request("http://example.com/")
self.assertRaises(urllib.error.URLError, o.open, req)
self.assertEqual(o.calls, [(handlers[0], "http_open", (req, ), {})])
def fetch(self, request, opener=None, summary=None):
if not self.__enable_http:
return (None, None)
if opener is None:
opener = OpenerDirector()
opener.add_handler(HTTPDefaultErrorHandler())
opener.add_handler(HTTPSHandler())
t = time.clock()
response = opener.open(request)
body = response.read()
t = timedelta(seconds=time.clock() - t)
url = request.get_full_url()
self.__context.get_logger().info('HTTP time: %s\n%s' % (t, url))
if self.__log_http:
log_dir = os.path.join(self.__context.get_config_dir(), 'http-log')
makedirs(log_dir)
log_file = os.path.join(log_dir,
datetime.utcnow().strftime(
'%Y-%m-%d-%H-%M-%S-%f'))
if summary is not None:
log_file += '-' + _safe_str(summary)
fp = open(log_file, 'w')
fp.write('\n\n'.join([
request.get_full_url(),
request.get_data() or 'No request data',
body or 'No response body',
]))
fp.close()
return (response, body)
def __init__(self, username, password=None):
"""Class to store info required to connect to the web server"""
# Get password if necessary
if password is None:
password = getpass()
# Get URL for the database
self.db_url = "http://virgodb.dur.ac.uk:8080/Eagle/"
# Set up authentication and cookies
self.password_mgr = HTTPPasswordMgrWithDefaultRealm()
self.password_mgr.add_password(None, self.db_url, username, password)
self.opener = OpenerDirector()
self.auth_handler = HTTPBasicAuthHandler(self.password_mgr)
self.cookie_handler = HTTPCookieProcessor(cookie_jar)
def build_opener(*handlers, **kw):
"""Create an opener object from a list of handlers.
The opener will use several default handlers, including support
for HTTP and FTP.
If any of the handlers passed as arguments are subclasses of the
default handlers, the default handlers will not be used.
"""
def isclass(obj):
return isinstance(obj, class_type_) or hasattr(obj, "__bases__")
opener = OpenerDirector()
default_classes = [
ProxyHandler, UnknownHandler, HTTPHandler, HTTPDefaultErrorHandler,
HTTPRedirectHandler, FTPHandler, FileHandler, HTTPErrorProcessor
]
check_classes = list(default_classes)
check_classes.append(HTTPSContextHandler)
skip = []
for klass in check_classes:
for check in handlers:
if isclass(check):
if issubclass(check, klass):
skip.append(klass)
elif isinstance(check, klass):
skip.append(klass)
for klass in default_classes:
if klass not in skip:
opener.add_handler(klass())
# Pick up SSL context from keyword settings
ssl_context = kw.get('ssl_context')
# Add the HTTPS handler with ssl_context
if HTTPSContextHandler not in skip:
opener.add_handler(HTTPSContextHandler(ssl_context))
for h in handlers:
if isclass(h):
h = h()
opener.add_handler(h)
return opener
def testKeywordArgs(self):
"""grabber.URLGrabber.__init__() **kwargs handling.
This is a simple test that just passes some arbitrary
values into the URLGrabber constructor and checks that
they've been set properly.
"""
opener = OpenerDirector()
g = URLGrabber(progress_obj=self.meter,
throttle=0.9,
bandwidth=20,
retry=20,
retrycodes=[5, 6, 7],
copy_local=1,
close_connection=1,
user_agent='test ua/1.0',
proxies={'http': 'http://www.proxy.com:9090'},
opener=opener)
opts = g.opts
self.assertEqual(opts.progress_obj, self.meter)
self.assertEqual(opts.throttle, 0.9)
self.assertEqual(opts.bandwidth, 20)
self.assertEqual(opts.retry, 20)
self.assertEqual(opts.retrycodes, [5, 6, 7])
self.assertEqual(opts.copy_local, 1)
self.assertEqual(opts.close_connection, 1)
self.assertEqual(opts.user_agent, 'test ua/1.0')
self.assertEqual(opts.proxies, {'http': 'http://www.proxy.com:9090'})
self.assertEqual(opts.opener, opener)
nopts = grabber.URLGrabberOptions(delegate=opts,
throttle=0.5,
copy_local=0)
self.assertEqual(nopts.progress_obj, self.meter)
self.assertEqual(nopts.throttle, 0.5)
self.assertEqual(nopts.bandwidth, 20)
self.assertEqual(nopts.retry, 20)
self.assertEqual(nopts.retrycodes, [5, 6, 7])
self.assertEqual(nopts.copy_local, 0)
self.assertEqual(nopts.close_connection, 1)
self.assertEqual(nopts.user_agent, 'test ua/1.0')
self.assertEqual(nopts.proxies, {'http': 'http://www.proxy.com:9090'})
nopts.opener = None
self.assertEqual(nopts.opener, None)
def build_opener(*handlers, **kw):
"""Create an opener object from a list of handlers.
The opener will use several default handlers, including support
for HTTP and FTP.
If any of the handlers passed as arguments are subclasses of the
default handlers, the default handlers will not be used.
"""
def isclass(obj):
return isinstance(obj, class_type_) or hasattr(obj, "__bases__")
opener = OpenerDirector()
default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
HTTPDefaultErrorHandler, HTTPRedirectHandler,
FTPHandler, FileHandler, HTTPErrorProcessor]
check_classes = list(default_classes)
check_classes.append(HTTPSContextHandler)
skip = []
for klass in check_classes:
for check in handlers:
if isclass(check):
if issubclass(check, klass):
skip.append(klass)
elif isinstance(check, klass):
skip.append(klass)
for klass in default_classes:
if klass not in skip:
opener.add_handler(klass())
# Pick up SSL context from keyword settings
ssl_context = kw.get('ssl_context')
# Add the HTTPS handler with ssl_context
if HTTPSContextHandler not in skip:
opener.add_handler(HTTPSContextHandler(ssl_context))
for h in handlers:
if isclass(h):
h = h()
opener.add_handler(h)
return opener
def test_proxy_https_proxy_authorization(self):
o = OpenerDirector()
ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128"))
o.add_handler(ph)
https_handler = MockHTTPSHandler()
o.add_handler(https_handler)
req = Request("https://www.example.com/")
req.add_header("Proxy-Authorization", "FooBar")
req.add_header("User-Agent", "Grail")
self.assertEqual(req.get_host(), "www.example.com")
self.assertTrue(req._tunnel_host is None)
r = o.open(req)
# Verify Proxy-Authorization gets tunneled to request.
# httpsconn req_headers do not have the Proxy-Authorization header but
# the req will have.
self.assertFalse(("Proxy-Authorization", "FooBar") in https_handler.httpconn.req_headers)
self.assertTrue(("User-Agent", "Grail") in https_handler.httpconn.req_headers)
self.assertFalse(req._tunnel_host is None)
self.assertEqual(req.get_host(), "proxy.example.com:3128")
self.assertEqual(req.get_header("Proxy-authorization"), "FooBar")
def test_proxy_basic_auth(self):
opener = OpenerDirector()
ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128"))
opener.add_handler(ph)
password_manager = MockPasswordManager()
auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager)
realm = "ACME Networks"
http_handler = MockHTTPHandler(
407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm)
opener.add_handler(auth_handler)
opener.add_handler(http_handler)
self._test_basic_auth(
opener,
auth_handler,
"Proxy-authorization",
realm,
http_handler,
password_manager,
"http://acme.example.com:3128/protected",
"proxy.example.com:3128",
)
def test5(url="http://example.com"):
# ruleid: insecure-openerdirector-open
OpenerDirector().open(url)
def resolve(self, item, captcha_cb=None, select_cb=None):
item = item.copy()
util.init_urllib()
url = self._url(item['url'])
page = ''
try:
opener = OpenerDirector()
opener.add_handler(HTTPHandler())
opener.add_handler(UnknownHandler())
install_opener(opener)
request = Request(url)
request.add_header('User-Agent', util.UA)
response = urlopen(request)
page = response.read()
response.close()
except HTTPError as e:
traceback.print_exc()
return
data = util.substr(page, '<form method=post target=\"iframe_dwn\"',
'</form>')
action = re.search('action=(?P<url>[^>]+)', data,
re.IGNORECASE | re.DOTALL)
img = re.search('<img src=\"(?P<url>[^\"]+)', data,
re.IGNORECASE | re.DOTALL)
if img and action:
sessid = []
for cookie in re.finditer('(PHPSESSID=[^\;]+)',
response.headers.get('Set-Cookie'),
re.IGNORECASE | re.DOTALL):
sessid.append(cookie.group(1))
# we have to download image ourselves
image = util.request(self._url(img.group('url')),
headers={
'Referer': url,
'Cookie': sessid[-1]
})
img_file = os.path.join(self.tmp_dir, 'captcha.png')
util.save_data_to_file(image, img_file)
code = None
if captcha_cb:
code = captcha_cb({'id': '0', 'img': img_file})
if not code:
self.info('No captcha received, exit')
return
request = urllib.urlencode({'code': code})
req = Request(self._url(action.group('url')), request)
req.add_header('User-Agent', util.UA)
req.add_header('Referer', url)
req.add_header('Cookie', sessid[-1])
try:
resp = urlopen(req)
if resp.code == 302:
file_url = resp.headers.get('location')
else:
file_url = resp.geturl()
if file_url.find(action.group('url')) > 0:
msg = resp.read()
resp.close()
js_msg = re.search('alert\(\'(?P<msg>[^\']+)', msg,
re.IGNORECASE | re.DOTALL)
if js_msg:
raise ResolveException(js_msg.group('msg'))
self.error(msg)
raise ResolveException(
'Nelze ziskat soubor, zkuste to znovu')
resp.close()
if file_url.find('data') >= 0 or file_url.find(
'download_free') > 0:
item['url'] = file_url
return item
self.error('wrong captcha, retrying')
return self.resolve(item, captcha_cb, select_cb)
except HTTPError:
traceback.print_exc()
return
raise HTTPError(full_url, 401, "Stopping Oauth2 failure loop for {}".format(full_url),
response.headers, response)
auth_token = self._get_outh2_token(www_authenticate_header)
request.add_unredirected_header('Authorization', 'Bearer ' + auth_token)
return self.parent.open(request, timeout=request.timeout)
# Got some help from this example https://gist.github.com/FiloSottile/2077115
class HeadRequest(Request):
def get_method(self):
return "HEAD"
better_urllib_get = OpenerDirector()
better_urllib_get.addheaders = DEFAULT_HEADERS.copy()
better_urllib_get.add_handler(HTTPHandler())
better_urllib_get.add_handler(HTTPSHandler())
better_urllib_get.add_handler(HTTPRedirectHandler())
better_urllib_get.add_handler(SocketFileHandler())
better_urllib_get.add_handler(Oauth2TokenAuthHandler())
class RegistryError(Exception):
def __init__(self, response):
self.response_obj = response
# Util functions
#############################################################################################
def build_test_opener(*handler_instances):
opener = OpenerDirector()
for h in handler_instances:
opener.add_handler(h)
return opener
super().__init__(host='', port=0, timeout=timeout)
self.socket_file = socket_file
def connect(self):
self.sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM, proto=0, fileno=None)
self.sock.settimeout(self.timeout)
self.sock.connect(self.socket_file)
def socket_open(self, req):
socket_file, path = req.selector.split(':')
req.host = socket_file
req.selector = path
return self.do_open(self.SocketFileToHttpConnectionAdaptor, req)
better_urllib = OpenerDirector()
better_urllib.addheaders = [DEFAULT_HEADERS]
better_urllib.add_handler(HTTPHandler())
better_urllib.add_handler(HTTPSHandler())
better_urllib.add_handler(SocketFileHandler())
# Util functions
#############################################################################################
def parse_thresholds(spec):
"""
Given a spec string break it up into ':' separated chunks. Convert strings to ints as it makes sense
:param spec:
:return: The thresholds, as intergers
"""
parts = spec.split(':')
def connect(self):
self.sock = socket.socket(family=socket.AF_UNIX,
type=socket.SOCK_STREAM,
proto=0,
fileno=None)
self.sock.settimeout(self.timeout)
self.sock.connect(self.socket_file)
def socket_open(self, req):
socket_file, path = req.selector.split(':', 1)
req.host = socket_file
req.selector = path
return self.do_open(self.SocketFileToHttpConnectionAdaptor, req)
better_urllib_get = OpenerDirector()
better_urllib_get.addheaders = DEFAULT_HEADERS.copy()
better_urllib_get.add_handler(HTTPHandler())
better_urllib_get.add_handler(HTTPSHandler())
better_urllib_get.add_handler(SocketFileHandler())
# Util functions
#############################################################################################
@lru_cache()
def get_url(url):
response = better_urllib_get.open(url, timeout=timeout)
return process_urllib_response(response), response.status
def test6_ok(url="https://example.com"):
od = OpenerDirector()
# ok: insecure-openerdirector-open
od.open(url)
def connect(self):
self.sock = socket.socket(family=socket.AF_UNIX,
type=socket.SOCK_STREAM,
proto=0,
fileno=None)
self.sock.settimeout(self.timeout)
self.sock.connect(self.socket_file)
def socket_open(self, req):
socket_file, path = req.selector.split(':')
req.host = socket_file
req.selector = path
return self.do_open(self.SocketFileToHttpConnectionAdaptor, req)
better_urllib = OpenerDirector()
better_urllib.addheaders = [DEFAULT_HEADERS]
better_urllib.add_handler(HTTPHandler())
better_urllib.add_handler(HTTPSHandler())
better_urllib.add_handler(SocketFileHandler())
# Util functions
#############################################################################################
def parse_thresholds(spec):
"""
Given a spec string break it up into ':' separated chunks. Convert strings to ints as it makes sense
:param spec:
:return: The thresholds, as intergers
"""
def test5_ok(url="https://example.com"):
# ok: insecure-openerdirector-open
OpenerDirector().open(url)
response.headers, response)
auth_token = self._get_outh2_token(www_authenticate_header)
request.add_unredirected_header('Authorization',
'Bearer ' + auth_token)
return self.parent.open(request, timeout=request.timeout)
# Got some help from this example https://gist.github.com/FiloSottile/2077115
class HeadRequest(Request):
def get_method(self):
return "HEAD"
better_urllib_get = OpenerDirector()
better_urllib_get.addheaders = DEFAULT_HEADERS.copy()
better_urllib_get.add_handler(HTTPHandler())
better_urllib_get.add_handler(HTTPRedirectHandler())
better_urllib_get.add_handler(SocketFileHandler())
better_urllib_get.add_handler(Oauth2TokenAuthHandler())
class RegistryError(Exception):
def __init__(self, response):
self.response_obj = response
# Util functions
#############################################################################################
def test6(url="http://example.com"):
od = OpenerDirector()
# ruleid: insecure-openerdirector-open
od.open(url)
class RequestHandler(BaseHTTPRequestHandler):
opener_director = OpenerDirector()
opener_director.add_handler(HTTPHandler())
server_version = 'DemoServer/0.1'
def do_POST(self):
global test_data
self.send_response(200)
arguments = json.loads(str(self.rfile.read(int(self.headers['Content-Length'])), 'ascii'))
traceparent = None
tracestate = Tracestate()
test_data.append({
'headers': self.get_headers('traceparent') + self.get_headers('tracestate'),
})
try:
temp_traceparent = BaseTraceparent.from_string(self.get_header('traceparent'))
if temp_traceparent.version == 0:
if temp_traceparent._residue:
raise ValueError('illegal traceparent format')
traceparent = Traceparent(0, temp_traceparent.trace_id, temp_traceparent.parent_id, temp_traceparent.trace_flags)
test_data[-1]['is_traceparent_valid'] = True
except ValueError:
test_data[-1]['is_traceparent_valid'] = False
try:
header = self.get_header('tracestate', commaSeparated = True)
if header:
tracestate = Tracestate(header)
if test_data[-1]['is_traceparent_valid']:
test_data[-1]['is_tracestate_valid'] = True
except ValueError:
# if tracestate is malformed, reuse the traceparent instead of restart the trace
# traceparent = Traceparent()
test_data[-1]['is_tracestate_valid'] = False
if traceparent is None:
# if traceparent is malformed, discard tracestate
traceparent = Traceparent()
tracestate = Tracestate()
for item in arguments:
headers = {}
headers['traceparent'] = str(Traceparent(0, traceparent.trace_id, None, traceparent.trace_flags))
if tracestate.is_valid():
headers['tracestate'] = str(tracestate)
request = Request(method = 'POST', url = item['url'], headers = headers, data = bytes(json.dumps(item['arguments']), 'ascii'))
with self.opener_director.open(request, timeout = self.timeout) as response:
pass
self.end_headers()
def get_headers(self, name):
name = name.lower()
headers = filter(lambda kv: kv[0].lower() == name, self.headers.items())
return tuple(headers)
def get_header(self, name, commaSeparated = False):
headers = self.get_headers(name)
# https://tools.ietf.org/html/rfc7230#section-3.2
# remove the leading whitespace and trailing whitespace
headers = map(lambda kv: kv[1].strip(' \t'), headers)
headers = tuple(headers)
if not headers:
return None
if commaSeparated:
return ','.join(headers)
if len(headers) == 1:
return headers[0]
raise ValueError('multiple header {} not allowed'.format(name))
def log_message(self, format, *args):
pass
def perform_request(url: str, opener: OpenerDirector = None):
if not opener:
opener = urllib.request.build_opener()
return opener.open(fullurl=url)
super().__init__(host='', port=0, timeout=timeout)
self.socket_file = socket_file
def connect(self):
self.sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM, proto=0, fileno=None)
self.sock.settimeout(self.timeout)
self.sock.connect(self.socket_file)
def socket_open(self, req):
socket_file, path = req.selector.split(':', 1)
req.host = socket_file
req.selector = path
return self.do_open(self.SocketFileToHttpConnectionAdaptor, req)
better_urllib_get = OpenerDirector()
better_urllib_get.addheaders = DEFAULT_HEADERS.copy()
better_urllib_get.add_handler(HTTPHandler())
better_urllib_get.add_handler(HTTPSHandler())
better_urllib_get.add_handler(SocketFileHandler())
# Util functions
#############################################################################################
@lru_cache()
def get_url(url):
response = better_urllib_get.open(url, timeout=timeout)
return process_urllib_response(response), response.status
def test1_ok():
od = OpenerDirector()
# ok: insecure-openerdirector-open
od.open("https://example.com")
def __init__(self):
OpenerDirector.__init__(self)
self.recorded = []
