def parse(self,content,p=None): '''parses a URL or a response-string to doc tree''' #HACK. With CSW data need to check for JSON/masked 429s first try: if p=='fromstring': etp = self._parse_f(content) #parse using string method else: etp = self._parse_p(content,p) #parse normally or using provided parser except u_lib.HTTPError as he: raise #but this won't happen because LXML pushes HTTP errors up to IO errors except IOError as ioe: if re.search('failed to load HTTP resource', ioe.message): raise u_lib.HTTPError(content, 429, 'IOE. Possible HTTP429 Rate Limiting Error. '+ioe.message, None, None) raise u_lib.HTTPError(content, 404, 'IOE. Probable HTTP Error. '+ioe.message, None, None) except Exception as e: raise return etp
def test_repeat_failure_general_http(self, mock_logging, mock_post): a = netuitive.Client(api_key='apikey') e = netuitive.Element() e.add_sample('nonsparseDataStrategy', 1434110794, 1, 'COUNTER', host='hostname') errs = [403, 429, 503, 404, 503, 204, 307, 302, 405, 413] for i in range(a.max_post_errors): mock_post.return_value = MockResponse(code=errs[i]) mock_post.side_effect = urllib2.HTTPError(a.url, errs[i], '', {}, None) resp = a.post(e) resp = a.post(e) self.assertNotEqual(resp, True) self.assertFalse(resp) self.assertFalse(a.disabled) self.assertEqual(len(e.samples), 0) self.assertEqual( mock_logging.exception.call_args_list[0][0][0], 'error posting payload to api ingest endpoint (%s): %s')
def test_not_kill_switch_504(self, mock_logging, mock_post): mock_post.return_value = MockResponse(code=504) a = netuitive.Client(api_key='apikey') mock_post.side_effect = urllib2.HTTPError(a.url, 504, '', {}, None) e = netuitive.Element() e.add_sample('nonsparseDataStrategy', 1434110794, 1, 'COUNTER', host='hostname') resp = a.post(e) resp2 = a.post(e) self.assertNotEqual(resp, True) self.assertFalse(resp2) self.assertFalse(a.disabled) self.assertEqual( mock_logging.exception.call_args_list[0][0][0], 'error posting payload to api ingest endpoint (%s): %s')
def test_kill_switch_410(self, mock_logging, mock_post): mock_post.return_value = MockResponse(code=410) a = netuitive.Client(api_key='apikey') mock_post.side_effect = urllib2.HTTPError(a.url, 410, '', {}, None) e = netuitive.Element() e.add_sample('nonsparseDataStrategy', 1434110794, 1, 'COUNTER', host='hostname') resp = a.post(e) resp2 = a.post(e) self.assertNotEqual(resp, True) self.assertFalse(resp2) self.assertTrue(a.disabled) self.assertEqual( mock_logging.exception.call_args_list[0][0][0], 'Posting has been disabled.See previous errors for details.')
def _submitRun(run, webclient, benchmark, counter=0): programTexts = [] for programPath in run.sourcefiles: with open(programPath, 'r') as programFile: programText = programFile.read() programTexts.append(programText) params = {'programText': programTexts} if benchmark.config.revision: params['svnBranch'] = _svn_branch params['revision'] = _svn_revision if run.propertyfile: with open(run.propertyfile, 'r') as propertyFile: propertyText = propertyFile.read() params['propertyText'] = propertyText limits = benchmark.rlimits if MEMLIMIT in limits: params['memoryLimitation'] = str(limits[MEMLIMIT]) + "MB" if TIMELIMIT in limits: params['timeLimitation'] = limits[TIMELIMIT] if CORELIMIT in limits: params['coreLimitation'] = limits[CORELIMIT] if benchmark.config.cpu_model: params['cpuModel'] = benchmark.config.cpu_model invalidOption = _handleOptions(run, params, limits) if invalidOption: raise WebClientError('Command {0} of run {1} contains option that is not usable with the webclient. '\ .format(run.options, run.identifier)) headers = { "Content-Type": "application/x-www-form-urlencoded", "Content-Encoding": "deflate", "Accept": "text/plain" } paramsCompressed = zlib.compress( urllib.urlencode(params, doseq=True).encode('utf-8')) request = urllib2.Request(webclient + "runs/", paramsCompressed, headers) # send request try: response = urllib2.urlopen(request) except urllib2.HTTPError as e: if (e.code == 401 and counter < 3): _auth(webclient, benchmark.config) return _submitRun(run, webclient, benchmark, counter + 1) else: raise e if response.getcode() == 200: runID = response.read() return runID else: raise urllib2.HTTPError(response.read(), response.getcode())
def test_get_urlopen_with_httperror(self, mock_urlopen): mock_urlopen.side_effect = urllib2.HTTPError( 'http://url.com', code=404, msg="Not Found", hdrs='', fp=None, ) ret_val = tvdb._get('http://url.com') self.assertEqual(ret_val, '')
def test_health_checks(self): self._mox.StubOutWithMock(urllib_request, 'urlopen') urllib_request.urlopen('http://localhost:%s/health' % self.PORT, None, timeout=1.0).AndReturn(OpenedURL('ok')) urllib_request.urlopen('http://localhost:%s/health' % self.PORT, None, timeout=1.0).AndReturn(OpenedURL('not ok')) urllib_request.urlopen('http://localhost:%s/health' % self.PORT, None, timeout=1.0).AndReturn( OpenedURL('not ok', code=200)) urllib_request.urlopen('http://localhost:%s/health' % self.PORT, None, timeout=1.0).AndReturn(OpenedURL('ok', code=400)) urllib_request.urlopen('http://localhost:%s/health' % self.PORT, None, timeout=1.0).AndRaise( urllib_request.HTTPError( '', 501, '', None, None)) urllib_request.urlopen('http://localhost:%s/health' % self.PORT, None, timeout=1.0).AndReturn(OpenedURL('ok', code=200)) urllib_request.urlopen('http://localhost:%s/random/endpoint' % self.PORT, None, timeout=1.0).AndReturn(OpenedURL('ok')) self._mox.ReplayAll() signaler = HttpSignaler(self.PORT) assert signaler('/health', expected_response='ok') == (True, None) assert signaler('/health', expected_response='ok') == ( False, 'Response differs from expected response (expected "ok", got "not ok")' ) assert signaler('/health', expected_response_code=200) == (True, None) assert signaler('/health', expected_response_code=200) == ( False, 'Response code differs from expected response (expected 200, got 400)' ) assert signaler('/health', expected_response_code=200) == ( False, 'Response code differs from expected response (expected 200, got 501)' ) assert signaler('/health', expected_response='ok', expected_response_code=200) == (True, None) assert signaler('/random/endpoint', expected_response='ok') == (True, None)
def _request(self, method, path, body={}, headers={}, expectedStatusCodes=[200], user_pwd=None): connection = self._get_connection() headers["Connection"] = "Keep-Alive" if user_pwd: base64_user_pwd = base64.b64encode( user_pwd.encode("utf-8")).decode("utf-8") headers["Authorization"] = "Basic " + base64_user_pwd elif self._base64_user_pwd: headers["Authorization"] = "Basic " + self._base64_user_pwd counter = 0 while (counter < 5): counter += 1 # send request try: connection.request(method, path, body=body, headers=headers) response = connection.getresponse() except Exception as e: if (counter < 5): logging.debug( "Exception during {} request to {}: {}".format( method, path, e)) # create new TCP connection and try to send the request connection.close() sleep(1) continue else: raise if response.status in expectedStatusCodes: return (response.read(), response.getcode()) else: message = "" if response.status == 401: message = 'Error 401: Permission denied. Please check the URL given to --cloudMaster and specify credentials if necessary.' elif response.status == 404: message = 'Error 404: Not found. Please check the URL given to --cloudMaster.' else: message += response.read().decode('UTF-8') logging.warn(message) raise urllib2.HTTPError(path, response.getcode(), message, response.getheaders(), None)
def json_data_fetsher(url: str) -> Dict[str, Any]: """Gets data from url and transfoms it to json, if possible""" try: page = request.urlopen(url) except request.HTTPError as error: raise request.HTTPError(url, error.code, "Not a Valid URL!", error.hdrs, error.fp) try: page_string = page.read() json_data = json.loads(page_string) return json_data except JSONDecodeError: raise JSONDecodeError("Could not extract JSON data from page")
def test_query_container(staged_env): """ tests the check that checks if containers have been promoted successfully :param staged_env: The staged_env fixture :return: None """ stage_info = staged_env with patch.object(url_lib, 'urlopen') as mock_urllib: # positive tests mock_urllib.return_value = True query_container_registry_promotion(stage_info=stage_info) # negative tests mock_urllib.side_effect = url_lib.HTTPError(*[None] * 5) with pytest.raises(AssertionError): query_container_registry_promotion(stage_info=stage_info)
def test_failure_general_http(self, mock_logging, mock_post): mock_post.return_value = MockResponse(code=500) # test infrastructure endpoint url creation a = netuitive.Client(api_key='apikey') mock_post.side_effect = urllib2.HTTPError(a.url, 500, '', {}, None) e = netuitive.Check('check', 'test', 60) resp = a.post_check(e) self.assertNotEqual(resp, True) self.assertEquals(mock_post.call_count, a.max_check_retry_count + 1) self.assertEqual(mock_logging.exception.call_args_list[0][0][0], 'HTTPError posting payload to api ingest endpoint (%s): %s')
def test_kill_switch_410(self, mock_logging, mock_post): mock_post.return_value = MockResponse(code=410) # test infrastructure endpoint url creation a = netuitive.Client(api_key='apikey') mock_post.side_effect = urllib2.HTTPError(a.url, 410, '', {}, None) c = netuitive.Check('check', 'test', 60) resp = a.post_check(c) resp2 = a.post_check(c) self.assertNotEqual(resp, True) self.assertFalse(resp2) self.assertTrue(a.disabled) self.assertEqual(mock_logging.exception.call_args_list[0][0][0], 'Posting has been disabled.See previous errors for details.')
def test_not_kill_switch_504(self, mock_logging, mock_post): mock_post.return_value = MockResponse(code=504) # test infrastructure endpoint url creation a = netuitive.Client(api_key='apikey') mock_post.side_effect = urllib2.HTTPError(a.url, 504, '', {}, None) c = netuitive.Check('check', 'test', 60) resp = a.post_check(c) resp2 = a.post_check(c) self.assertNotEqual(resp, True) self.assertFalse(resp2) self.assertFalse(a.disabled) self.assertEqual(mock_logging.exception.call_args_list[0][0][0], 'HTTPError posting payload to api ingest endpoint (%s): %s')
def test_failure_general_http(self, mock_logging, mock_post): mock_post.return_value = MockResponse(code=500) # test infrastructure endpoint url creation a = netuitive.Client(api_key='apikey') mock_post.side_effect = urllib2.HTTPError(a.url, 500, '', {}, None) e = netuitive.Event( 'test', 'INFO', 'test event', 'big old test message', 'INFO') resp = a.post_event(e) self.assertNotEqual(resp, True) self.assertEqual(mock_logging.exception.call_args_list[0][0][ 0], 'error posting payload to api ingest endpoint (%s): %s')
def test_failure_general_http(self, mock_logging, mock_post): mock_post.return_value = MockResponse(code=500) # test infrastructure endpoint url creation a = netuitive.Client(api_key='apikey') mock_post.side_effect = urllib2.HTTPError(a.url, 500, '', {}, None) e = netuitive.Element() e.add_sample( 'nonsparseDataStrategy', 1434110794, 1, 'COUNTER', host='hostname') resp = a.post(e) self.assertNotEqual(resp, True) self.assertEqual(mock_logging.exception.call_args_list[0][0][ 0], 'error posting payload to api ingest endpoint (%s): %s')
def redirect_request(self, req, fp, code, msg, headers, newurl): newurl = newurl.replace(' ', '%20') if code in (301, 307): return request.Request(newurl, data=req.get_data(), headers=req.headers, origin_req_host=req.get_origin_req_host(), unverifiable=True) elif code in (302, 303): newheaders = dict( (k, v) for k, v in req.headers.items() if k.lower() not in ("content-length", "content-type")) return request.Request(newurl, headers=newheaders, origin_req_host=req.get_origin_req_host(), unverifiable=True) else: raise request.HTTPError(req.get_full_url(), code, msg, headers, fp)
def __request(method, path, body, headers, expectedStatusCodes=[200]): """ Sends a request specified by the parameters. @raise urlib.request.HTTPError: if the request fails more than 5 times @return: (response body, status code) """ connection = __get_connection() headers["Connection"] = "Keep-Alive" if __base64_user_pwd: headers["Authorization"] = "Basic " + __base64_user_pwd counter = 0 while (counter < 5): counter += 1 # send request try: connection.request(method, path, body=body, headers=headers) response = connection.getresponse() except: if (counter < 5): # create new TCP connection and try to send the request connection.close() sleep(1) continue else: raise if response.status in expectedStatusCodes: return (response.read(), response.getcode()) else: message = "" if response.status == 401: message = 'Please check the user and password given to --cloudUser.\n' if response.status == 404: message = 'Please check the URL given to --cloudMaster.' message += response.read().decode('UTF-8') raise request.HTTPError(path, response.getcode(), message, response.getheaders(), None)
def _request(method, path, body, headers, expectedStatusCodes=[200]): connection = _get_connection() headers["Connection"] = "Keep-Alive" if _base64_user_pwd: headers["Authorization"] = "Basic " + _base64_user_pwd counter = 0 while (counter < 5): counter+=1 # send request try: connection.request(method, path, body=body, headers=headers) response = connection.getresponse() except Exception as e: if (counter < 5): logging.debug("Exception during {} request to {}: {}".format(method, path, e)) # create new TCP connection and try to send the request connection.close() sleep(1) continue else: raise if response.status in expectedStatusCodes: return (response.read(), response.getcode()) else: message = "" if response.status == 401: message = 'Please check the URL given to --cloudMaster.\n' if response.status == 404: message = 'Please check the URL given to --cloudMaster.' message += response.read().decode('UTF-8') logging.warn(message) raise urllib2.HTTPError(path, response.getcode(), message , response.getheaders(), None)
def download(url, filename): if subprocess.call(['curl', url, '-o', filename]) != 0: raise request.HTTPError('Download failed.')
def http_error_default(self, req, fp, code, msg, headers): result = urlreq.HTTPError(req.get_full_url(), code, msg, headers, fp) result.status = code return result
def _parse_f(self,content): res = u_lib.urlopen(content).read() if re.search('API rate limit exceeded',res): raise u_lib.HTTPError(content, 429, 'Masked HTTP429 Rate Limiting Error. ', None, None) return etree.fromstring(res).getroottree()
def http_error_503(self, req, fp, code, msg, headers): raise urllib2.HTTPError(req, code, throttled_message, headers, fp)
def http_error(code): return urllib2.HTTPError('mock-url', code, 'error', {}, None)