def parse(self,content,p=None):
'''parses a URL or a response-string to doc tree'''
#HACK. With CSW data need to check for JSON/masked 429s first
try:
if p=='fromstring': etp = self._parse_f(content) #parse using string method
else: etp = self._parse_p(content,p) #parse normally or using provided parser
except u_lib.HTTPError as he:
raise #but this won't happen because LXML pushes HTTP errors up to IO errors
except IOError as ioe:
if re.search('failed to load HTTP resource', ioe.message):
raise u_lib.HTTPError(content, 429, 'IOE. Possible HTTP429 Rate Limiting Error. '+ioe.message, None, None)
raise u_lib.HTTPError(content, 404, 'IOE. Probable HTTP Error. '+ioe.message, None, None)
except Exception as e:
raise
return etp
def test_repeat_failure_general_http(self, mock_logging, mock_post):
a = netuitive.Client(api_key='apikey')
e = netuitive.Element()
e.add_sample('nonsparseDataStrategy',
1434110794,
1,
'COUNTER',
host='hostname')
errs = [403, 429, 503, 404, 503, 204, 307, 302, 405, 413]
for i in range(a.max_post_errors):
mock_post.return_value = MockResponse(code=errs[i])
mock_post.side_effect = urllib2.HTTPError(a.url, errs[i], '', {},
None)
resp = a.post(e)
resp = a.post(e)
self.assertNotEqual(resp, True)
self.assertFalse(resp)
self.assertFalse(a.disabled)
self.assertEqual(len(e.samples), 0)
self.assertEqual(
mock_logging.exception.call_args_list[0][0][0],
'error posting payload to api ingest endpoint (%s): %s')
def test_not_kill_switch_504(self, mock_logging, mock_post):
mock_post.return_value = MockResponse(code=504)
a = netuitive.Client(api_key='apikey')
mock_post.side_effect = urllib2.HTTPError(a.url, 504, '', {}, None)
e = netuitive.Element()
e.add_sample('nonsparseDataStrategy',
1434110794,
1,
'COUNTER',
host='hostname')
resp = a.post(e)
resp2 = a.post(e)
self.assertNotEqual(resp, True)
self.assertFalse(resp2)
self.assertFalse(a.disabled)
self.assertEqual(
mock_logging.exception.call_args_list[0][0][0],
'error posting payload to api ingest endpoint (%s): %s')
def test_kill_switch_410(self, mock_logging, mock_post):
mock_post.return_value = MockResponse(code=410)
a = netuitive.Client(api_key='apikey')
mock_post.side_effect = urllib2.HTTPError(a.url, 410, '', {}, None)
e = netuitive.Element()
e.add_sample('nonsparseDataStrategy',
1434110794,
1,
'COUNTER',
host='hostname')
resp = a.post(e)
resp2 = a.post(e)
self.assertNotEqual(resp, True)
self.assertFalse(resp2)
self.assertTrue(a.disabled)
self.assertEqual(
mock_logging.exception.call_args_list[0][0][0],
'Posting has been disabled.See previous errors for details.')
def _submitRun(run, webclient, benchmark, counter=0):
programTexts = []
for programPath in run.sourcefiles:
with open(programPath, 'r') as programFile:
programText = programFile.read()
programTexts.append(programText)
params = {'programText': programTexts}
if benchmark.config.revision:
params['svnBranch'] = _svn_branch
params['revision'] = _svn_revision
if run.propertyfile:
with open(run.propertyfile, 'r') as propertyFile:
propertyText = propertyFile.read()
params['propertyText'] = propertyText
limits = benchmark.rlimits
if MEMLIMIT in limits:
params['memoryLimitation'] = str(limits[MEMLIMIT]) + "MB"
if TIMELIMIT in limits:
params['timeLimitation'] = limits[TIMELIMIT]
if CORELIMIT in limits:
params['coreLimitation'] = limits[CORELIMIT]
if benchmark.config.cpu_model:
params['cpuModel'] = benchmark.config.cpu_model
invalidOption = _handleOptions(run, params, limits)
if invalidOption:
raise WebClientError('Command {0} of run {1} contains option that is not usable with the webclient. '\
.format(run.options, run.identifier))
headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Content-Encoding": "deflate",
"Accept": "text/plain"
}
paramsCompressed = zlib.compress(
urllib.urlencode(params, doseq=True).encode('utf-8'))
request = urllib2.Request(webclient + "runs/", paramsCompressed, headers)
# send request
try:
response = urllib2.urlopen(request)
except urllib2.HTTPError as e:
if (e.code == 401 and counter < 3):
_auth(webclient, benchmark.config)
return _submitRun(run, webclient, benchmark, counter + 1)
else:
raise e
if response.getcode() == 200:
runID = response.read()
return runID
else:
raise urllib2.HTTPError(response.read(), response.getcode())
def test_get_urlopen_with_httperror(self, mock_urlopen):
mock_urlopen.side_effect = urllib2.HTTPError(
'http://url.com',
code=404,
msg="Not Found",
hdrs='',
fp=None,
)
ret_val = tvdb._get('http://url.com')
self.assertEqual(ret_val, '')
def test_health_checks(self):
self._mox.StubOutWithMock(urllib_request, 'urlopen')
urllib_request.urlopen('http://localhost:%s/health' % self.PORT,
None,
timeout=1.0).AndReturn(OpenedURL('ok'))
urllib_request.urlopen('http://localhost:%s/health' % self.PORT,
None,
timeout=1.0).AndReturn(OpenedURL('not ok'))
urllib_request.urlopen('http://localhost:%s/health' % self.PORT,
None,
timeout=1.0).AndReturn(
OpenedURL('not ok', code=200))
urllib_request.urlopen('http://localhost:%s/health' % self.PORT,
None,
timeout=1.0).AndReturn(OpenedURL('ok',
code=400))
urllib_request.urlopen('http://localhost:%s/health' % self.PORT,
None,
timeout=1.0).AndRaise(
urllib_request.HTTPError(
'', 501, '', None, None))
urllib_request.urlopen('http://localhost:%s/health' % self.PORT,
None,
timeout=1.0).AndReturn(OpenedURL('ok',
code=200))
urllib_request.urlopen('http://localhost:%s/random/endpoint' %
self.PORT,
None,
timeout=1.0).AndReturn(OpenedURL('ok'))
self._mox.ReplayAll()
signaler = HttpSignaler(self.PORT)
assert signaler('/health', expected_response='ok') == (True, None)
assert signaler('/health', expected_response='ok') == (
False,
'Response differs from expected response (expected "ok", got "not ok")'
)
assert signaler('/health', expected_response_code=200) == (True, None)
assert signaler('/health', expected_response_code=200) == (
False,
'Response code differs from expected response (expected 200, got 400)'
)
assert signaler('/health', expected_response_code=200) == (
False,
'Response code differs from expected response (expected 200, got 501)'
)
assert signaler('/health',
expected_response='ok',
expected_response_code=200) == (True, None)
assert signaler('/random/endpoint',
expected_response='ok') == (True, None)
def _request(self,
method,
path,
body={},
headers={},
expectedStatusCodes=[200],
user_pwd=None):
connection = self._get_connection()
headers["Connection"] = "Keep-Alive"
if user_pwd:
base64_user_pwd = base64.b64encode(
user_pwd.encode("utf-8")).decode("utf-8")
headers["Authorization"] = "Basic " + base64_user_pwd
elif self._base64_user_pwd:
headers["Authorization"] = "Basic " + self._base64_user_pwd
counter = 0
while (counter < 5):
counter += 1
# send request
try:
connection.request(method, path, body=body, headers=headers)
response = connection.getresponse()
except Exception as e:
if (counter < 5):
logging.debug(
"Exception during {} request to {}: {}".format(
method, path, e))
# create new TCP connection and try to send the request
connection.close()
sleep(1)
continue
else:
raise
if response.status in expectedStatusCodes:
return (response.read(), response.getcode())
else:
message = ""
if response.status == 401:
message = 'Error 401: Permission denied. Please check the URL given to --cloudMaster and specify credentials if necessary.'
elif response.status == 404:
message = 'Error 404: Not found. Please check the URL given to --cloudMaster.'
else:
message += response.read().decode('UTF-8')
logging.warn(message)
raise urllib2.HTTPError(path, response.getcode(), message,
response.getheaders(), None)
def json_data_fetsher(url: str) -> Dict[str, Any]:
"""Gets data from url and transfoms it to json, if possible"""
try:
page = request.urlopen(url)
except request.HTTPError as error:
raise request.HTTPError(url, error.code, "Not a Valid URL!",
error.hdrs, error.fp)
try:
page_string = page.read()
json_data = json.loads(page_string)
return json_data
except JSONDecodeError:
raise JSONDecodeError("Could not extract JSON data from page")
def test_query_container(staged_env):
"""
tests the check that checks if containers have been promoted successfully
:param staged_env: The staged_env fixture
:return: None
"""
stage_info = staged_env
with patch.object(url_lib, 'urlopen') as mock_urllib:
# positive tests
mock_urllib.return_value = True
query_container_registry_promotion(stage_info=stage_info)
# negative tests
mock_urllib.side_effect = url_lib.HTTPError(*[None] * 5)
with pytest.raises(AssertionError):
query_container_registry_promotion(stage_info=stage_info)
def test_failure_general_http(self, mock_logging, mock_post):
mock_post.return_value = MockResponse(code=500)
# test infrastructure endpoint url creation
a = netuitive.Client(api_key='apikey')
mock_post.side_effect = urllib2.HTTPError(a.url, 500, '', {}, None)
e = netuitive.Check('check', 'test', 60)
resp = a.post_check(e)
self.assertNotEqual(resp, True)
self.assertEquals(mock_post.call_count, a.max_check_retry_count + 1)
self.assertEqual(mock_logging.exception.call_args_list[0][0][0], 'HTTPError posting payload to api ingest endpoint (%s): %s')
def test_kill_switch_410(self, mock_logging, mock_post):
mock_post.return_value = MockResponse(code=410)
# test infrastructure endpoint url creation
a = netuitive.Client(api_key='apikey')
mock_post.side_effect = urllib2.HTTPError(a.url, 410, '', {}, None)
c = netuitive.Check('check', 'test', 60)
resp = a.post_check(c)
resp2 = a.post_check(c)
self.assertNotEqual(resp, True)
self.assertFalse(resp2)
self.assertTrue(a.disabled)
self.assertEqual(mock_logging.exception.call_args_list[0][0][0], 'Posting has been disabled.See previous errors for details.')
def test_not_kill_switch_504(self, mock_logging, mock_post):
mock_post.return_value = MockResponse(code=504)
# test infrastructure endpoint url creation
a = netuitive.Client(api_key='apikey')
mock_post.side_effect = urllib2.HTTPError(a.url, 504, '', {}, None)
c = netuitive.Check('check', 'test', 60)
resp = a.post_check(c)
resp2 = a.post_check(c)
self.assertNotEqual(resp, True)
self.assertFalse(resp2)
self.assertFalse(a.disabled)
self.assertEqual(mock_logging.exception.call_args_list[0][0][0], 'HTTPError posting payload to api ingest endpoint (%s): %s')
def test_failure_general_http(self, mock_logging, mock_post):
mock_post.return_value = MockResponse(code=500)
# test infrastructure endpoint url creation
a = netuitive.Client(api_key='apikey')
mock_post.side_effect = urllib2.HTTPError(a.url, 500, '', {}, None)
e = netuitive.Event(
'test', 'INFO', 'test event', 'big old test message', 'INFO')
resp = a.post_event(e)
self.assertNotEqual(resp, True)
self.assertEqual(mock_logging.exception.call_args_list[0][0][
0], 'error posting payload to api ingest endpoint (%s): %s')
def test_failure_general_http(self, mock_logging, mock_post):
mock_post.return_value = MockResponse(code=500)
# test infrastructure endpoint url creation
a = netuitive.Client(api_key='apikey')
mock_post.side_effect = urllib2.HTTPError(a.url, 500, '', {}, None)
e = netuitive.Element()
e.add_sample(
'nonsparseDataStrategy', 1434110794, 1, 'COUNTER', host='hostname')
resp = a.post(e)
self.assertNotEqual(resp, True)
self.assertEqual(mock_logging.exception.call_args_list[0][0][
0], 'error posting payload to api ingest endpoint (%s): %s')
def redirect_request(self, req, fp, code, msg, headers, newurl):
newurl = newurl.replace(' ', '%20')
if code in (301, 307):
return request.Request(newurl,
data=req.get_data(),
headers=req.headers,
origin_req_host=req.get_origin_req_host(),
unverifiable=True)
elif code in (302, 303):
newheaders = dict(
(k, v) for k, v in req.headers.items()
if k.lower() not in ("content-length", "content-type"))
return request.Request(newurl,
headers=newheaders,
origin_req_host=req.get_origin_req_host(),
unverifiable=True)
else:
raise request.HTTPError(req.get_full_url(), code, msg, headers, fp)
def __request(method, path, body, headers, expectedStatusCodes=[200]):
"""
Sends a request specified by the parameters.
@raise urlib.request.HTTPError: if the request fails more than 5 times
@return: (response body, status code)
"""
connection = __get_connection()
headers["Connection"] = "Keep-Alive"
if __base64_user_pwd:
headers["Authorization"] = "Basic " + __base64_user_pwd
counter = 0
while (counter < 5):
counter += 1
# send request
try:
connection.request(method, path, body=body, headers=headers)
response = connection.getresponse()
except:
if (counter < 5):
# create new TCP connection and try to send the request
connection.close()
sleep(1)
continue
else:
raise
if response.status in expectedStatusCodes:
return (response.read(), response.getcode())
else:
message = ""
if response.status == 401:
message = 'Please check the user and password given to --cloudUser.\n'
if response.status == 404:
message = 'Please check the URL given to --cloudMaster.'
message += response.read().decode('UTF-8')
raise request.HTTPError(path, response.getcode(), message,
response.getheaders(), None)
def _request(method, path, body, headers, expectedStatusCodes=[200]):
connection = _get_connection()
headers["Connection"] = "Keep-Alive"
if _base64_user_pwd:
headers["Authorization"] = "Basic " + _base64_user_pwd
counter = 0
while (counter < 5):
counter+=1
# send request
try:
connection.request(method, path, body=body, headers=headers)
response = connection.getresponse()
except Exception as e:
if (counter < 5):
logging.debug("Exception during {} request to {}: {}".format(method, path, e))
# create new TCP connection and try to send the request
connection.close()
sleep(1)
continue
else:
raise
if response.status in expectedStatusCodes:
return (response.read(), response.getcode())
else:
message = ""
if response.status == 401:
message = 'Please check the URL given to --cloudMaster.\n'
if response.status == 404:
message = 'Please check the URL given to --cloudMaster.'
message += response.read().decode('UTF-8')
logging.warn(message)
raise urllib2.HTTPError(path, response.getcode(), message , response.getheaders(), None)
def download(url, filename):
if subprocess.call(['curl', url, '-o', filename]) != 0:
raise request.HTTPError('Download failed.')
def http_error_default(self, req, fp, code, msg, headers):
result = urlreq.HTTPError(req.get_full_url(), code, msg, headers, fp)
result.status = code
return result
def _parse_f(self,content):
res = u_lib.urlopen(content).read()
if re.search('API rate limit exceeded',res):
raise u_lib.HTTPError(content, 429, 'Masked HTTP429 Rate Limiting Error. ', None, None)
return etree.fromstring(res).getroottree()
def http_error_503(self, req, fp, code, msg, headers):
raise urllib2.HTTPError(req, code, throttled_message, headers, fp)
def http_error(code):
return urllib2.HTTPError('mock-url', code, 'error', {}, None)
