def getHtml(self, url, hdr=None):
data = None
try:
if not hdr:
req = Request(url, data, headers)
else:
req = Request(url, data, hdr)
response = urlopen(req, timeout=60)
data = response.read()
response.close()
except urllib2.HTTPError as e:
data = e.read()
raise urllib2.HTTPError()
return data
def retry(self, eStr=""):
if self.current_attemp < self.attemps_limit:
self.current_attemp += 1
self.status = "ready"
self.shared_var.value = 0
self.thread_shared_cmds = {}
self.start()
else:
s = 'The maximum retry attempts reached'
if eStr:
s += " (%s)" % eStr
self.errors.append(
urllib2.HTTPError(self.url, "0", s, {}, StringIO()))
self._failed = True
def abrir_url(url):
#print "A fazer request de: " + url
try:
req = urllib2.Request(url)
req.add_header('User-Agent', user_agent)
response = urllib2.urlopen(req)
link = response.read()
response.close()
return link
except urllib2.HTTPError, e:
mensagemok(
'wareztuga.tv',
str(urllib2.HTTPError(e.url, e.code, traducao(40199), e.hdrs,
e.fp)), traducao(40200))
sys.exit(0)
def open(self, request):
"""Logs the request and returns a MockResponse object."""
full_url = request.get_full_url()
if "?" in full_url:
url = full_url[:full_url.find("?")]
else:
url = full_url
if (url != "https://www.google.com/accounts/ClientLogin"
and not url.endswith("_ah/login")):
assert "X-appcfg-api-version" in request.headers
assert "User-agent" in request.headers
request_data = (full_url, bool(request.data))
self.requests.append(request_data)
if self.cookie:
request.headers["Cookie"] = self.cookie
response = self.responses[url](request)
# Use ordered responses in preference to specific response to generic 200.
if url in self.ordered_responses:
logging.debug("Using ordered pre-canned response for: %s" %
full_url)
response = self.ordered_responses[url].pop(0)(request)
if not self.ordered_responses[url]:
self.ordered_responses.pop(url)
elif url in self.responses:
logging.debug("Using pre-canned response for: %s" % full_url)
response = self.responses[url](request)
elif self.strict:
raise Exception('No response found for url: %s (%s)' %
(url, full_url))
else:
logging.debug("Using generic blank response for: %s" %
full_url)
response = TestRpcServerMixin.MockResponse("")
if "Set-Cookie" in response.headers:
self.cookie = response.headers["Set-Cookie"]
# Handle error status codes in the same way as the appengine_rpc openers.
# urllib2 will raise HTTPError for non-2XX status codes, per RFC 2616.
if not (200 <= response.code < 300):
code, msg, hdrs = response.code, response.msg, response.info()
fp = StringIO.StringIO(response.read())
raise urllib2.HTTPError(url=url,
code=code,
msg=None,
hdrs=hdrs,
fp=fp)
return response
def get(self, url):
# 1. Request target resource
# 2. Redirect to SSO Service
res = self.opener.open(url)
# Already logged in? (directly got to requesting page)
if res.geturl() == url:
return res
# Prepare authentication payload. Implementation specific!
self.ensure_credentials()
auth_url = res.geturl()
auth_data = urllib.urlencode({
'j_username': self.username,
'j_password': self.password
})
# 3. Request SSO Service
# 4. Respond with XHTML form
res = self.opener.open(auth_url, auth_data)
# Authentication failed? (staying on same page)
if res.geturl() == auth_url:
raise urllib2.HTTPError(auth_url, 401, 'Authentication failed', res.info(), res)
# Prepare assertion payload. Implementation specific!
content = res.read()
html_parser = HTMLParser.HTMLParser()
assertion_url_regex = re.compile('<form action="(.*?)" method="post">')
relay_state_regex = re.compile('<input type="hidden" name="RelayState" value="(.*?)"/>')
saml_response_regex = re.compile('<input type="hidden" name="SAMLResponse" value="(.*?)"/>')
assertion_url_match = assertion_url_regex.search(content)
relay_state_match = relay_state_regex.search(content)
saml_response_match = saml_response_regex.search(content)
assertion_url = html_parser.unescape(assertion_url_match.group(1))
relay_state = html_parser.unescape(relay_state_match.group(1))
saml_response = saml_response_match.group(1)
saml_data = urllib.urlencode({
'RelayState': relay_state,
'SAMLResponse': saml_response
})
# 5. Request Assertion Consumer Service
# 6. Redirect to target resource
# 7. Request target resource
# 8. Respond with requested resource
return self.opener.open(assertion_url, saml_data)
def find_user_password(self, realm, authuri):
""" Limit number of queries per request.
Note that retries needs to be reset in the calling code.
"""
# allow sending the username:password 5 times before failing!
if self.retries > 5:
from httplib import HTTPMessage
from StringIO import StringIO
raise urllib2.HTTPError(authuri, 401,
"basic auth failed for realm %r" % realm,
HTTPMessage(StringIO("")), None)
self.retries += 1
return urllib2.HTTPPasswordMgrWithDefaultRealm.find_user_password(
self, realm, authuri)
def testErrorWithContent(self):
status = remote.RpcStatus(state=remote.RpcState.REQUEST_ERROR,
error_message='an error')
urllib2.urlopen(mox.Func(self.VerifyRequest)).AndRaise(
urllib2.HTTPError(
'http://whatever', 500, 'An error occured',
{'content-type': 'application/json'},
StringIO.StringIO(protojson.encode_message(status))))
self.mox.ReplayAll()
rpc = self.trans.send_rpc(my_method.remote, self.request)
rpc.wait()
self.assertEquals(remote.RpcState.REQUEST_ERROR, rpc.state)
self.assertEquals('an error', rpc.error_message)
self.assertEquals(None, rpc.error_name)
def redirect_request(self, req, fp, code, msg, headers, redirected_url):
if code in (301, 302, 303, 307):
redirected_url = redirected_url.replace(' ', '%20')
newheaders = dict((k,v) for k,v in req.headers.items()
if k.lower() not in ("content-length", "content-type"))
warn_msg = "Got a " + str(code) + " redirection (" + redirected_url + ")."
print settings.print_warning_msg(warn_msg)
return HeadRequest(redirected_url,
headers = newheaders,
origin_req_host = req.get_origin_req_host(),
unverifiable = True
)
else:
err_msg = str(urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)).replace(": "," (")
print settings.print_critical_msg(err_msg + ").")
raise SystemExit()
def testUploadRetryErrors(self, side_effect=None):
"""Verify that we retry errors (and eventually give up)"""
if not side_effect:
side_effect = urllib2.HTTPError('http://', 400, 'fail', {}, None)
self.upload_mock.side_effect = side_effect
errors = ctypes.c_int()
item = upload_symbols.FakeItem(sym_file='/dev/null')
element = upload_symbols.SymbolElement(item, None)
ret = upload_symbols.UploadSymbol(self.url,
element,
'TestProduct',
num_errors=errors)
self.assertEqual(ret, 1)
self.upload_mock.assert_called_with(self.url, item, 'TestProduct')
self.assertTrue(
self.upload_mock.call_count >= upload_symbols.MAX_RETRIES)
def test_fetch(self, m_mkdir, m_open, m_urlopen, m_exists):
m_exists.return_value = True
m_urlopen().read.side_effect = ["foo", "bar", ""]
pathname = self.i.fetch("/does_not_exist")
self.assertEqual(
pathname,
"/does_not_exist/user-distro-release-arch.28e5ebabd9d8f6e237df63da2b503785093f0229241bc7021198f63c43b93269.qcow2"
)
self.assertEqual(m_open().write.call_args_list,
[mock.call("foo"), mock.call("bar")])
m_exists.return_value = False
m_urlopen.side_effect = urllib2.HTTPError(*[None] * 5)
self.assertRaises(error.FetchFailedException, self.i.fetch,
"/does_not_exist")
self.assertEqual(m_mkdir.call_args_list,
[mock.call("/does_not_exist")])
def http_error_auth_reqed(self, auth_header, host, req, headers):
authreq = headers.get(auth_header, None)
if self.retried > 5:
# Don't fail endlessly - if we failed once, we'll probably
# fail a second time. Hm. Unless the Password Manager is
# prompting for the information. Crap. This isn't great
# but it's better than the current 'repeat until recursion
# depth exceeded' approach <wink>
raise urllib2.HTTPError(req.get_full_url(), 401,
"digest auth failed", headers, None)
else:
self.retried += 1
if authreq:
scheme = authreq.split()[0]
if scheme.lower() == 'x-digest':
return self.retry_http_digest_auth(req, authreq)
def download(_from, _to):
u = urlparse.urlparse(_from)
if not u.hostname:
raise urllib2.HTTPError('%s is not a valid URL.' % _from)
try:
u = urllib2.urlopen(_from)
with open(_to, 'w') as out:
while True:
buffer = u.read(block_sz)
if not buffer:
break
out.write(buffer)
return True
except urllib2.HTTPError as e:
print '%s : %s' % (_from, e)
raise e
def abrir_url_cookie(url, parametros=None):
from t0mm0.common.net import Net
net = Net()
net.set_cookies(cookie_wt)
try:
if parametros:
link = net.http_POST(url, parametros).content.encode(
'latin-1', 'ignore')
link = net.http_GET(url).content.encode('latin-1', 'ignore')
return link
except urllib2.HTTPError, e:
mensagemok(
'wareztuga.tv',
str(urllib2.HTTPError(e.url, e.code, traducao(40199), e.hdrs,
e.fp)), traducao(40200))
sys.exit(0)
def _upload_with_chunked_encoding(self,
method,
url,
query_params,
fileobj,
progress_callback=None):
# Start the request.
parsed_base_url = urlparse.urlparse(self._base_url)
path = url + '?' + urllib.urlencode(query_params)
if parsed_base_url.scheme == 'http':
conn = httplib.HTTPConnection(parsed_base_url.netloc)
else:
conn = httplib.HTTPSConnection(parsed_base_url.netloc)
with closing(conn):
conn.putrequest(method, parsed_base_url.path + path)
# Set headers.
headers = {
'Authorization': 'Bearer ' + self._get_access_token(),
'Transfer-Encoding': 'chunked',
'X-Requested-With': 'XMLHttpRequest',
}
headers.update(self._extra_headers)
for header_name, header_value in headers.iteritems():
conn.putheader(header_name, header_value)
conn.endheaders()
# Use chunked transfer encoding to send the data through.
bytes_uploaded = 0
while True:
to_send = fileobj.read(16 * 1024)
if not to_send:
break
conn.send('%X\r\n%s\r\n' % (len(to_send), to_send))
bytes_uploaded += len(to_send)
if progress_callback is not None:
progress_callback(bytes_uploaded)
conn.send('0\r\n\r\n')
# Read the response.
response = conn.getresponse()
if response.status != 200:
# Low-level httplib module doesn't throw HTTPError
raise urllib2.HTTPError(self._base_url + path, response.status,
response.reason,
dict(response.getheaders()),
StringIO(response.read()))
def http_error_401(self, req, fp, code, msg, headers):
try:
if self._retried > 5:
raise urllib2.HTTPError(req.get_full_url(), 401,
"negotiate auth failed", headers, None)
self._retried += 1
logging.debug("gssapi handler, try %s" % self._retried)
negotiate = get_negociate_value(headers)
if negotiate is None:
logging.debug(
"no negociate found in a www-authenticate header")
return None
logging.debug("HTTPGssapiAuthHandler: negotiate 1 is %r" %
negotiate)
result, self._context = krb.authGSSClientInit("HTTP@%s" %
req.get_host())
if result < 1:
raise GssapiAuthError(
"HTTPGssapiAuthHandler: init failed with %d" % result)
result = krb.authGSSClientStep(self._context, negotiate)
if result < 0:
raise GssapiAuthError(
"HTTPGssapiAuthHandler: step 1 failed with %d" % result)
client_response = krb.authGSSClientResponse(self._context)
logging.debug("HTTPGssapiAuthHandler: client response is %s..." %
client_response[:10])
req.add_unredirected_header("Authorization",
"Negotiate %s" % client_response)
server_response = self.parent.open(req)
negotiate = get_negociate_value(server_response.info())
if negotiate is None:
logging.warning(
"HTTPGssapiAuthHandler: failed to authenticate server")
else:
logging.debug("HTTPGssapiAuthHandler negotiate 2: %s" %
negotiate)
result = krb.authGSSClientStep(self._context, negotiate)
if result < 1:
raise GssapiAuthError(
"HTTPGssapiAuthHandler: step 2 failed with %d" %
result)
return server_response
except GssapiAuthError as exc:
logging.error(repr(exc))
finally:
self.clean_context()
self._reset()
def non_recursive_http_error_auth_reqed(self, authreq, host, req, headers):
authreq = headers.get(authreq, None)
if not hasattr(self, 'retried'):
self.retried = 0
if self.retried > 5:
raise urllib2.HTTPError(req.get_full_url(), 401, "basic auth failed", headers, None)
else:
self.retried += 1
if authreq:
mo = urllib2.AbstractBasicAuthHandler.rx.search(authreq)
if mo:
scheme, quote, realm = mo.groups()
if scheme.lower() == 'basic':
return self.retry_http_basic_auth(host, req, realm)
def http_error_auth_reqed(self, auth_header_field, url, req, headers):
if self.retried > 3:
# Don't fail endlessly - if we failed once, we'll probably
# fail a second time. Hm. Unless the Password Manager is
# prompting for the information. Crap. This isn't great
# but it's better than the current 'repeat until recursion
# depth exceeded' approach <wink>
raise urllib2.HTTPError(req.get_full_url(), 401,
"NTLM auth failed", headers, None)
else:
self.retried += 1
auth_header_value = headers.get(auth_header_field, None)
if auth_header_field and 'ntlm' in auth_header_value.lower():
return self.retry_using_http_NTLM_auth(req, auth_header_field,
None, headers)
def test_create(self):
io = StringIO('{ "measurements": [ 123456789, 123456790 ] }')
with mock.patch('urllib2.urlopen', return_value=io):
self.assertEqual(self.cmd.create(), 123456789)
io = StringIO('{ "measurements": 123456789 }')
with mock.patch('urllib2.urlopen', return_value=io):
self.assertEqual(self.cmd.create(), False)
io = StringIO('{ "testing": 123456789 }')
with mock.patch('urllib2.urlopen', return_value=io):
self.assertEqual(self.cmd.create(), False)
seffect = urllib2.HTTPError("test_url", 500,
StringIO("Server is down"),
StringIO('testing'), StringIO('testing'))
with mock.patch('urllib2.urlopen', side_effect=seffect):
print self.cmd.create()
def MapMatch(self,Coordinates,Times) :
"""
Coordinates ==> coordonnees en 4326 des points
Times ==> TimeStemps des points (dans l'ordre bien sur)
"""
#rajouter un point au debut et a la fin pour avoir le bon nombre de point
Coordinates = [Coordinates[0]]+Coordinates+[Coordinates[-1]]
Times = [Times[0]-1]+Times+[Times[-1]+1]
Coords = ";".join([str(e) for e in Coordinates]).replace(")","").replace("(","").replace(" ","")
TimeStamps = ";".join([str(e) for e in Times])
Request = self.Root+"/match/v1/bike/"+Coords+"?geometries=geojson&overview=full&annotations=true×tamps="+TimeStamps
try :
Json = urllib2.urlopen(Request).read()
except urllib2.HTTPError :
print(Request)
raise urllib2.HTTPError("The request failed")
return Json
def redirect_request(self, req, fp, code, msg, headers, newurl):
newurl = newurl.replace(' ', '%20')
if code in (301, 307):
return urllib2.Request(newurl,
data=req.get_data(),
headers=req.headers,
origin_req_host=req.get_origin_req_host(),
unverifiable=True)
elif code in (302, 303):
newheaders = dict((k, v) for k, v in req.headers.items()
if k.lower() not in ("content-length", "content-type"))
return urllib2.Request(newurl,
headers=newheaders,
origin_req_host=req.get_origin_req_host(),
unverifiable=True)
else:
raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
def test_http_error_status(self):
source = {
'key': 'TypeformAPIKey',
'forms': [{'id': 'someid', 'name': 'Test Survey'}]
}
# mock the HTTPError that should be returned from the server
code = 403
body = BytesIO(json.dumps({'status': code}))
err = urllib2.HTTPError('url', code, 'Forbidden', {}, body)
urllib2.urlopen = MagicMock(side_effect=err)
stream = Typeform(source, OPTIONS)
try:
stream.read()
except TypeformError, e:
self.assertEqual(str(e), 'HTTP StatusCode ' + str(code))
def test_proxy_request_http_error(self, stderr, Request, urlopen):
lines = []
stderr.write.side_effect = lambda data, **kw: lines.append(data)
request = mock.Mock()
Request.return_value = request
fp = mock.Mock()
fp.read.return_value = "something went wrong"
urlopen.side_effect = urllib2.HTTPError("/", 400, "Bad request", {}, fp)
with self.assertRaises(SystemExit) as cm:
admin_plugin.proxy_request(self.service_name, "/", body="waat",
method="GET")
exc = cm.exception
self.assertEqual(1, exc.code)
expected_output = r"""ERROR: 400 - Bad request
something went wrong
"""
self.assertEqual(expected_output, "".join(lines))
def abrir_url(url, erro=True):
try:
Debug("A fazer request normal de: " + url)
req = urllib2.Request(url)
req.add_header('User-Agent', user_agent)
response = urllib2.urlopen(req)
link = response.read()
response.close()
return link
except urllib2.HTTPError, e:
host = 'http://' + url.split('/')[2]
xbmcgui.Dialog().ok(
'TV Portuguesa',
str(
urllib2.HTTPError(e.url, e.code, "Erro na página.", e.hdrs,
e.fp)),
'Verifique manualmente se a página está operacional.', host)
sys.exit(0)
def test_error(self):
context = self.layer['folder']
request = self.layer['request']
self._expect_request().throw(
urllib2.HTTPError('url', 500, 'Internal Server Error', None, None))
self.replay()
view = getMultiAdapter((context, request), name='watch')
view()
self.assertEqual(request.response.headers.get('location'),
context.absolute_url())
messages = IStatusMessage(request).show()
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0].message,
u'The dashboard portlet could not be created.')
def test_wait_for_http_httperror(self):
with patch('verify_mediawiki_bundle._get_ssl_ctx',
autospec=True,
return_value=None) as ssl_mock:
with patch('urllib2.urlopen',
autospec=True,
return_value=urllib2.HTTPError(None, None, None, None,
None)) as uo_mock:
with patch('verify_mediawiki_bundle.until_timeout',
autospec=True,
return_value=[1]) as ut_mock:
with self.assertRaisesRegexp(
JujuAssertionError,
'example.com is not reachable'):
wait_for_http("example.com")
uo_mock.assert_called_once_with('example.com')
self.assertEqual(ut_mock.mock_calls, [call(600)])
ssl_mock.assert_called_once_with()
def test_send_query(self):
host = 'example.com'
line = '/line?param=1'
url = 'http://%s%s' % (host, line)
with LogDir() as log_dir:
KM.init('key', host=host, to_stderr=False, log_dir=log_dir)
urlopen = self.mocker.replace('urllib2.urlopen')
# First
urlopen(url)
self.mocker.result(None)
# Second
urlopen(url)
self.mocker.throw(urllib2.HTTPError(url, 500, 'Error', None, None))
with self.mocker:
# First
KM.send_query(line)
# Second
self.assertRaises(urllib2.HTTPError, KM.send_query, line)
def test_raise_http_request(self, urlopen):
error = http.HTTPError('http://example.com/', 404, 'Not Found', None,
None)
error.read = lambda: b'o'
class _Fake(object):
def close(self):
return 0
class _Faker(object):
_closer = _Fake()
error.file = _Faker()
urlopen.side_effect = error
resp, content = OAuthRemoteApp.http_request('http://example.com')
assert resp.code == 404
assert b'o' in content
def jsonToPython(self, response_string, response):
""" Convert a json string to Python objects ... handle known instances
where server injects badly formed JSON into the stream
"""
if 'DOCTYPE HTML PUBLIC' in response_string:
errmsg = 'SERVER ERROR : '
if 'server encountered an internal error' in response_string:
errmsg += 'server encountered an unspecified internal error.'
ecode = 503
else:
ecode = 500
errmsg += 'server returned HTML, not valid JSON.\n'
errmsg += response_string
raise urllib2.HTTPError(response.geturl(), ecode, errmsg, None,
None)
server_error = 'SERVER ERROR : '
errors = []
if '[Failure instance:' in response_string:
found_start = response_string.find('[Failure instance:')
while found_start > 0:
found_end = response_string.find('\n],', found_start)
error = response_string[found_start:found_end + 3]
errors.append(''.join(error.splitlines()))
before = response_string[:found_start]
after = response_string[found_end + 3:]
response_string = before + after
found_start = response_string.find('[Failure instance:')
if errors:
errmsg = 'the following errors found in returned JSON string :'
print server_error, errmsg
for error in errors:
print error
print 'The resulting data block may be incomplete.'
sys.stdout.flush()
try:
return json.loads(response_string)
except Exception as e:
errmsg += 'unable to handle improperly formated JSON from server.\n'
errmsg += response.geturl() + '\n'
errmsg += 'Returned JSON = ' + response_string
setattr(e, 'details', errmsg)
raise e
def redirect_request(self, req, fp, code, msg, headers, newurl):
"""Return a Request or None in response to a redirect.
The default response in redirect_request claims not to
follow directives in RFC 2616 but in fact it does
This class does not and makes handling 302 with POST
possible
"""
m = req.get_method()
if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
or code in (301, 302, 303) and m == "POST"):
newurl = newurl.replace(' ', '%20')
return urllib2.Request(newurl,
data=req.get_data(),
headers=req.headers,
origin_req_host=req.get_origin_req_host(),
unverifiable=True)
else:
raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
def test_HTTPError(self, mock_urlopen, mock_request):
config = configobj.ConfigObj()
config['host'] = '127.0.0.1'
config['port'] = '4242'
metric = Metric('servers.myhostname.cpu.cpu_count',
123,
raw_value=123,
timestamp=1234567,
host='myhostname',
metric_type='GAUGE')
handler = TSDBHandler(config)
header = {'Content-Type': 'application/json'}
exception = urllib2.HTTPError(url=self.url,
code=404,
msg="Error",
hdrs=header,
fp=None)
handler.side_effect = exception
handler.process(metric)
