def test_retry_vs_no_retry(self): def error(*args, **kwargs): t.append('e') raise e hb = self.new_heartbeat_task() hb._urllib2.urlopen = error hb._retry_gap_secs = 1 hb.set_interval_secs(100) hb._heartbeat_max_retries = 2 # http non-5xx t = [] e = HTTPError(None, 400, None, None, None) hb.tick() self.assertEquals(t, ['e']) # http 5xx t = [] e = HTTPError(None, 500, None, None, None) hb.tick() self.assertEquals(t, ['e', 'e']) # urlerror t = [] e = URLError("foo") hb.tick() self.assertEquals(t, ['e', 'e']) # httpexception t = [] e = HTTPException() hb.tick() self.assertEquals(t, ['e', 'e'])
def get(self, project_id): ''' Browser download of a project file ''' pdb = Projects() project = pdb.get(project_id) if project['filename']: filename = os.path.join(self.get_project_dir(), project['filename']) if os.path.exists(filename): proj_file = file(filename, 'rb') self.set_header('content_type', 'application/octet-stream') self.set_header('Content-Length', str(os.path.getsize(filename))) form_proj = clean_filename(project['projectname']) form_ver = clean_filename(project['version']) form_date = strftime('%Y-%m-%d_%H%M%S') self.set_header('Content-Disposition', 'attachment; filename=%s-%s-%s.proj' % (form_proj, form_ver, form_date)) try: self.write(proj_file.read()) finally: proj_file.close() else: raise HTTPError(filename, 403, "%s is not a file" % filename, None, None) else: raise HTTPError(filename, 403, "no file found for %s" % \ project['projectname'], None, None)
def _rss_failure(self): http_error = HTTPError('http://rss.test.com', 500, 'Unauthorized', None, None) http_error.read = MagicMock() http_error.read.return_value = json.dumps({'exceptionId': 'SVC1006'}) self._exp_mock.set_provider_limit.side_effect = http_error
def redirect_request(self, req, fp, code, msg, headers, newurl): """ Method that will be called by urllib2 when a redirection occur, to see what to do. """ m = req.get_method() if ((code in (301, 302, 303, 307) and m in ("GET", "HEAD")) or (code in (300, 302, 303) and m == "POST")): newurl = newurl.replace(' ', '%20') oldurl = req.get_full_url() newApiCall = newurl.split('/')[-1].split('?')[0] oldApiCall = oldurl.split('/')[-1].split('?')[0] # # # print 'newurl:',newurl # print 'req.data:',req.data # print 'req.get_full_url():', req.get_full_url() # print # # if oldApiCall != newApiCall: msg = "Not allowed redirection: trying to change data service" msg += " API call from '%s' to '%s'" % (oldApiCall, newApiCall) raise HTTPError(req.get_full_url(), code, msg, headers, fp) return Request(newurl, req.data, headers = req.headers, origin_req_host = req.get_origin_req_host(), unverifiable = True) else: raise HTTPError(req.get_full_url(), code, msg, headers, fp)
def _rss_failure(self): http_error = HTTPError('http://rss.test.com', 500, 'Unauthorized', None, None) http_error.read = MagicMock() http_error.read.return_value = json.dumps({ 'exceptionId': 'SVC1006' }) self._exp_mock.set_provider_limit.side_effect = http_error
def __fetch_url(self, url=None): try: resp = urlopen(url) data = resp.read() return dict(code=resp.code,data=data) except HTTPError, e: if e.code == 401: if self.environment == 'bws': raise HTTPError(url,e.code,"Your application is not approved yet",None,None) else: raise HTTPError(url,e.code,"The request requires user authentication",None,None)
def lookup_id(self, id): params = {'name': id} volumes = self.request('/volumes', params=params) found = None for volume in volumes: if volume['status'] != 'DELETED': if found: raise HTTPError('unused', 409, 'Conflict', {}, StringIO('{"reason": "conflict"}')) found = volume if not found: raise HTTPError('unused', 404, 'Not Found', {}, StringIO('{"reason": "not found"}')) return found['id']
def http_error_401(self, req, fp, code, msg, headers): """Override the default autoretry behaviour""" url = req.get_full_url() hdrs = req.header_items() hdrs = ', '.join( ['%s: %s' % (key, value) for key, value in sorted(hdrs)]) context = (url, hdrs) if context in self.authContext: self.authContext.clear() result = HTTPError(req.get_full_url(), code, msg, headers, fp) result.status = code return result self.authContext.add(context) return self.http_error_auth_reqed('www-authenticate', url, req, headers)
def test_500_error_server_error(self): message = PMMail(sender='*****@*****.**', to='*****@*****.**', subject='Subject', text_body='Body', api_key='test') with mock.patch('postmark.core.urlopen', side_effect=HTTPError('', 500, '', {}, None)): self.assertRaises(PMMailServerErrorException, message.send)
def testProperties3(self): c = HTTPError("http://localhost/", self.code, self.msg, None, None) e = dalq.DALServiceError(self.msg, self.code, c) self.assertEquals(self.msg, e.reason) self.assert_(e.cause is c) self.assertEquals(self.code, e.code) self.assert_(e.url is None)
def test_send_with_template(self): # Both template_id and template_model are set, so send should work. message = PMMail(api_key='test', sender='*****@*****.**', to='*****@*****.**', template_id=1, template_model={'junk': 'more junk'}) with mock.patch('postmark.core.urlopen', side_effect=HTTPError('', 200, '', {}, None)): message.send()
def search(self, what, start=0): """Search for what starting on specified page. Defaults to first page of results.""" logging.debug("parse_search({}, {})".format(what, start)) # If we're on first page of results, we'll try to find other pages if start == 0: self.first_page = True else: self.first_page = False try: response = self.engine.opener.open('{}?nm={}&start={}'.format( self.engine.search_url, quote(what), start)) # Only continue if response status is OK. if response.getcode() != 200: raise HTTPError( response.geturl(), response.getcode(), "HTTP request to {} failed with status: {}".format( self.engine.search_url, response.getcode()), response.info(), None) except (URLError, HTTPError) as e: logging.error(e) raise e # Decode data and feed it to parser data = response.read().decode('cp1251') data = re.sub(r'<wbr>|<b>|<\/b>', '', data) self.feed(data)
def __init__(self): """Initialize Pornolab search engine, signing in using given credentials.""" # Initialize various objects. self.cj = cookielib.CookieJar() self.opener = build_opener(HTTPCookieProcessor(self.cj)) self.url = 'https://pornolab.net' # Override url with the actual URL to be used (in case official URL isn't accessible) self.credentials = credentials # Add submit button additional POST param. self.credentials['login'] = u'Вход' try: logging.info("Trying to connect using given credentials.") response = self.opener.open( self.login_url, urlencode(dict_encode(self.credentials)).encode()) # Check if response status is OK. if response.getcode() != 200: raise HTTPError( response.geturl(), response.getcode(), "HTTP request to {} failed with status: {}".format( self.login_url, response.getcode()), response.info(), None) # Check if login was successful using cookies. if not 'bb_data' in [cookie.name for cookie in self.cj]: logging.debug(self.cj) raise ValueError("Unable to connect using given credentials.") else: logging.info("Login successful.") except (URLError, HTTPError, ValueError) as e: logging.error(e)
def redirect_request(self, req, fp, code, msg, headers, newurl): """Return a Request or None in response to a redirect. This is called by the http_error_30x methods when a redirection response is received. If a redirection should take place, return a new Request to allow http_error_30x to perform the redirect. Otherwise, raise HTTPError if no-one else should try to handle this url. Return None if you can't but another Handler might. """ m = req.get_method() if (code in (301, 302, 303, 307, "refresh") and m in ("GET", "HEAD") or code in (301, 302, 303, "refresh") and m == "POST"): # Strictly (according to RFC 2616), 301 or 302 in response # to a POST MUST NOT cause a redirection without confirmation # from the user (of urllib2, in this case). In practice, # essentially all clients do redirect in this case, so we do # the same. # TODO: really refresh redirections should be visiting; tricky to fix new = _request.Request( newurl, headers=req.headers, origin_req_host=req.get_origin_req_host(), unverifiable=True, visit=False, timeout=req.timeout) new._origin_req = getattr(req, "_origin_req", req) return new else: raise HTTPError(req.get_full_url(), code, msg, headers, fp)
def test_url_request_helper_unauthorized_error(mocker, caplog): caplog.set_level(logging.WARNING) config_section = mount_efs.CONFIG_SECTION config_item = mount_efs.DISABLE_FETCH_EC2_METADATA_TOKEN_ITEM config = get_config(config_section, config_item, "true") get_aws_ec2_metadata_token_mock = mocker.patch( "mount_efs.get_aws_ec2_metadata_token" ) url_open_mock = mocker.patch( "mount_efs.urlopen", side_effect=HTTPError("url", 401, "Unauthorized", None, None), ) resp = mount_efs.url_request_helper( config, mount_efs.INSTANCE_METADATA_SERVICE_URL, "", "" ) assert None == resp utils.assert_called(url_open_mock) utils.assert_not_called(get_aws_ec2_metadata_token_mock) assert "Unauthorized request" in [rec.message for rec in caplog.records][0] assert ( "ec2 metadata token is disabled" in [rec.message for rec in caplog.records][0] )
def test_proper_error_message_displayed_when_login_failed(self): # Arrange self.fs.CreateFile('nut_shell/shell.yml', contents=""" shell: name: nut_shell """) self.fs.CreateFile('nut_shell/cloudshell_config.yml', contents=""" install: host: localhost port: 9000 username: YOUR_USERNAME password: YOUR_PASSWORD domain: Global """) os.chdir('nut_shell') mock_installer = Mock() mock_installer.install = Mock( side_effect=HTTPError('', 401, LOGIN_ERROR_MESSAGE, None, None)) command_executor = InstallCommandExecutor(installer=mock_installer) # Act with self.assertRaises(FatalError) as context: command_executor.install() # Assert self.assertTrue( str(context.exception) == u'Login to CloudShell failed. Please verify the credentials in the config' )
def getImage(self, key): if key not in self.img: if self._d == 'g': try: #grv譜面は使いまわしされない画像かもしれない url = self.url[key] if key == 'bg': url = self.url['bg'][:-6] + 'gbg.png' elif key == 'bar': url = self.url['bar'][:-7] + 'gbar.png' imgdata = urllib2.urlopen(url).read() self.img[key] = Image.open(StringIO(imgdata)) self.url[key] = url except HTTPError: url = self.url[key] imgdata = urllib2.urlopen(url).read() self.img[key] = Image.open(StringIO(imgdata)) else: #レーンが消える背景はgbg.png try: url = self.url[key] imgdata = urllib2.urlopen(url).read() self.img[key] = Image.open(StringIO(imgdata)) except HTTPError: if key == 'bg': self.url['bg'] = self.url['bg'][:-6] + 'gbg.png' else: raise HTTPError(str(key) + 'のurlが不正です') url = self.url[key] imgdata = urllib2.urlopen(url).read() self.img[key] = Image.open(StringIO(imgdata)) return self.img[key]
def test_422_error_unprocessable_entity(self): messages = [ PMMail( sender="*****@*****.**", to="*****@*****.**", subject="Subject", text_body="Body", api_key="test", ), PMMail( sender="*****@*****.**", to="*****@*****.**", subject="Subject", text_body="Body", api_key="test", ), ] json_payload = BytesIO() json_payload.write(b'{"Message": "", "ErrorCode": 422}') json_payload.seek(0) batch = PMBatchMail(messages=messages, api_key="test") with mock.patch("postmark.core.urlopen", side_effect=HTTPError("", 422, "", {}, json_payload)): self.assertRaises(PMMailUnprocessableEntityException, batch.send)
def test_proper_error_message_when_non_authentication_http_error_raised( self): # Arrange self.fs.CreateFile('nut_shell/shell.yml', contents=""" shell: name: nut_shell """) self.fs.CreateFile('nut_shell/cloudshell_config.yml', contents=""" install: host: localhost port: 9000 username: YOUR_USERNAME password: YOUR_PASSWORD domain: Global """) os.chdir('nut_shell') mock_installer = Mock() mock_installer.install = Mock( side_effect=HTTPError('', 404, LOGIN_ERROR_MESSAGE, None, None)) command_executor = InstallCommandExecutor(installer=mock_installer) # Act with self.assertRaises(FatalError) as context: command_executor.install() # Assert self.assertTrue( str(context.exception) == u"Failed to install shell. CloudShell responded with: '{}'".format( 'Login failed for user: YOUR_USERNAME. ' 'Please make sure the username and password are correct.'), "Actual: {}".format(context.exception))
def __init__(self): """Initialize rutracker search engine, signing in using given credentials.""" # Initialize cookie handler. self.cj = cookielib.CookieJar() self.opener = build_opener(HTTPCookieProcessor(self.cj)) self.credentials = credentials # Add submit button additional POST param. self.credentials['login'] = u'Вход' # Send POST information and sign in. try: logging.info("Trying to connect using given credentials.") response = self.opener.open( self.login_url, urlencode(dict_encode(self.credentials)).encode()) # Check if response status is OK. if response.getcode() != 200: raise HTTPError( response.geturl(), response.getcode(), "HTTP request to {} failed with status: {}".format( self.login_url, response.getcode()), response.info(), None) # Check if login was successful using cookies. if not 'bb_data' in [cookie.name for cookie in self.cj]: raise ValueError("Unable to connect using given credentials.") else: logging.info("Login successful.") except (URLError, HTTPError, ValueError) as e: logging.error(e)
def redirect_request(self, newurl, req, fp, code, msg, headers): """Return a Request or None in response to a redirect. This is called by the http_error_30x methods when a redirection response is received. If a redirection should take place, return a new Request to allow http_error_30x to perform the redirect; otherwise, return None to indicate that an HTTPError should be raised. """ if code in (301, 302, 303, "refresh") or \ (code == 307 and not req.has_data()): # Strictly (according to RFC 2616), 301 or 302 in response to # a POST MUST NOT cause a redirection without confirmation # from the user (of urllib2, in this case). In practice, # essentially all clients do redirect in this case, so we do # the same. # XXX really refresh redirections should be visiting; tricky to # fix, so this will wait until post-stable release new = Request(newurl, headers=req.headers, origin_req_host=req.get_origin_req_host(), unverifiable=True, visit=False, ) new._origin_req = getattr(req, "_origin_req", req) return new else: raise HTTPError(req.get_full_url(), code, msg, headers, fp)
def _test_get_aws_security_credentials_instance_metadata_role_name(mocker, is_name_str=True, is_imds_v2=False): mocker.patch.dict(os.environ, {}) mocker.patch('os.path.exists', return_value=False) response = json.dumps({ 'Code': 'Success', 'LastUpdated': '2019-10-25T14:41:42Z', 'Type': 'AWS-HMAC', 'AccessKeyId': ACCESS_KEY_ID_VAL, 'SecretAccessKey': SECRET_ACCESS_KEY_VAL, 'Token': SESSION_TOKEN_VAL, 'Expiration': '2019-10-25T21:17:24Z' }) if is_name_str: role_name_data = b'FAKE_IAM_ROLE_NAME' else: role_name_data = 'FAKE_IAM_ROLE_NAME' if is_imds_v2: side_effects = [HTTPError('url', 401, 'Unauthorized', None, None)] mocker.patch('watchdog.get_aws_ec2_metadata_token', return_value='ABCDEFG==') else: side_effects = [] side_effects = side_effects + [MockUrlLibResponse(data=role_name_data), MockUrlLibResponse(data=response)] mocker.patch('watchdog.urlopen', side_effect=side_effects) credentials = watchdog.get_aws_security_credentials('metadata:') assert credentials['AccessKeyId'] == ACCESS_KEY_ID_VAL assert credentials['SecretAccessKey'] == SECRET_ACCESS_KEY_VAL assert credentials['Token'] == SESSION_TOKEN_VAL
def side_effect(*args, **kwargs): response = StringIO(u''' <title>Facebook | Error</title> Sorry, something went wrong. ''') http_exception = HTTPError('bla', 505, 'bla', 'bla', response) raise http_exception
def test_application_error(self): ''' Facebook errors often look like 500s Its a silly system, but we need to support it This is actually an application error ''' graph = self.guy.graph() with mock.patch('urllib2.build_opener') as patched: opener = mock.MagicMock() response = StringIO('''{ "error": { "type": "OAuthException", "message": "Error validating access token: USER_ID has not authorized application APP_ID" } }''') opener.open.side_effect = HTTPError('bla', 500, 'bla', 'bla', response) patched.return_value = opener def make_request(): graph.get('me') self.assertRaises(facebook_exceptions.OAuthException, make_request)
def test_getFileByAliasLibrarianLongServerError(self): # The Librarian server can return a 500 HTTPError. # LibrarienClient.getFileByAlias() returns a LibrarianServerError # if the server returns this error for a longer time than given # by the parameter timeout. _File = client_module._File client_module._File = make_mock_file( HTTPError('http://fake.url/', 500, 'Forced error', None, None), 2) client = InstrumentedLibrarianClient() alias_id = client.addFile('sample.txt', 6, StringIO('sample'), 'text/plain') transaction.commit() self.assertRaises(LibrarianServerError, client.getFileByAlias, alias_id, 1) client_module._File = make_mock_file(URLError('Connection refused'), 2) client = InstrumentedLibrarianClient() alias_id = client.addFile('sample.txt', 6, StringIO('sample'), 'text/plain') transaction.commit() self.assertRaises(LibrarianServerError, client.getFileByAlias, alias_id, 1) client_module._File = _File
def http_error_401(self, req, fp, code, msg, headers): """Override the default autoretry behaviour""" url = req.get_full_url() hdrs = req.header_items() hdrs = ', '.join(['%s: %s' % (key, value) for key, value in sorted(hdrs)]) context = (url, hdrs) if context in self.authContext: self.authContext.clear() result = HTTPError( req.get_full_url(), code, msg, headers, fp) result.status = code return result self.authContext.add(context) return self.http_error_auth_reqed('www-authenticate', url, req, headers)
def test_get_target_region_token_endpoint_not_allowed(mocker): get_aws_ec2_metadata_token_mock = mocker.patch( 'mount_efs.get_aws_ec2_metadata_token', side_effect=HTTPError('url', 405, 'Not allowed', None, None)) mocker.patch('mount_efs.urlopen', return_value=MockUrlLibResponse()) assert 'us-east-1' == get_target_region_helper() utils.assert_not_called(get_aws_ec2_metadata_token_mock)
def download_torrent(self, url): """Download file at url and write it to a file, print the path to the file and the url.""" # Make temp file. file, path = tempfile.mkstemp('.torrent') file = os.fdopen(file, "wb") # Set up fake POST params, needed to trick the server into sending the file. id = re.search(r'dl\.php\?t=(\d+)', url).group(1) post_params = { 't': id, } # Download torrent file at url. try: response = self.opener.open( url, urlencode(dict_encode(post_params)).encode()) # Only continue if response status is OK. if response.getcode() != 200: raise HTTPError( response.geturl(), response.getcode(), "HTTP request to {} failed with status: {}".format( url, response.getcode()), response.info(), None) except (URLError, HTTPError) as e: logging.error(e) raise e # Write it to a file. data = response.read() file.write(data) file.close() # Print file path and url. print(path + " " + url)
def redirect_request(self, req, fp, code, msg, headers, newurl): """ Apply redirect logic to a request. See parent HTTPRedirectHandler.redirect_request for parameter info. If the redirect is disallowed, this raises the corresponding HTTP error. If the redirect can't be determined, return None to allow other handlers to try. If the redirect is allowed, return the new request. This method specialized for the case when (a) the user knows that the redirect will not cause unacceptable side effects for any request method, and (b) the user knows that any request data should be passed through to the redirect. If either condition is not met, this should not be used. """ # note that requests being provided by a handler will use get_method to # indicate the method, by monkeypatching this, instead of setting the # Request object's method attribute. m = getattr(req, "method", req.get_method()) if not (code in (301, 302, 303, 307) and m in ("GET", "HEAD") or code in (301, 302, 303) and m in ("POST", "PUT")): raise HTTPError(req.full_url, code, msg, headers, fp) new_request = Request( newurl.replace(' ', '%20'), # space escaping in new url if needed. headers=req.headers, origin_req_host=req.origin_req_host, unverifiable=True, data=req.data, ) if PYTHON27_OR_OLDER: # the `method` attribute did not exist for Request in Python 2.7. new_request.get_method = lambda: m else: new_request.method = m return new_request
def redirect_request(self, req, fp, code, msg, headers, newurl): """Return a Request or None in response to a redirect. This is called by the http_error_30x methods when a redirection response is received. If a redirection should take place, return a new Request to allow http_error_30x to perform the redirect. Otherwise, raise HTTPError if no-one else should try to handle this url. Return None if you can't but another Handler might. """ m = req.get_method() if (code in (301, 302, 303, 307) and m in ("GET", "HEAD") or code in (301, 302, 303) and m == "POST"): # Strictly (according to RFC 2616), 301 or 302 in response # to a POST MUST NOT cause a redirection without confirmation # from the user (of urllib2, in this case). In practice, # essentially all clients do redirect in this case, so we # do the same. # be conciliant with URIs containing a space newurl = newurl.replace(' ', '%20') newheaders = dict((k, v) for k, v in req.headers.items() if k.lower() not in ( "content-length", "content-type") ) return urllib2.Request(newurl, headers=newheaders, origin_req_host=req.get_origin_req_host(), unverifiable=True) else: raise HTTPError(req.get_full_url(), code, msg, headers, fp)
def get_item_by_path(self, path, include_download_info=False): parameters = self.prepare_parameters() if path[-1:] == '/': path = path[:-1] Logger.debug(path + ' <- Target') key = '%s%s' % (self._driveid, path,) Logger.debug('Testing item from cache: %s' % key) item = self._items_cache.get(key) if not item: parameters['fields'] = 'files(%s)' % self._get_field_parameters() index = path.rfind('/') filename = urllib.unquote(path[index+1:]) parent = path[0:index] if not parent: parent = 'root' else: parent = self.get_item_by_path(parent, include_download_info)['id'] item = None parameters['q'] = '\'%s\' in parents and name = \'%s\'' % (Utils.str(parent), Utils.str(filename).replace("'","\\'")) files = self.get('/files', parameters = parameters) if (len(files['files']) > 0): for f in files['files']: item = self._extract_item(f, include_download_info) break else: Logger.debug('Found in cache.') if not item: raise RequestException('Not found by path', HTTPError(path, 404, 'Not found', None, None), 'Request URL: %s' % path, None) else: self._items_cache.set(key, item) return item
def fetch(self, url): self.log.debug("fetching HTML from %s", url) request = Request(url) request.add_header('User-Agent', self.userAgent) request.add_header('Referer', self._win.url) if self._win.doc.cookie: request.add_header('Cookie', self._win.doc.cookie) response = urlopen(request) if response.code != 200: self.log.warn("fail to fetch HTML from %s, code=%d, msg=%s", url, response.code, response.msg) raise HTTPError(url, response.code, "fail to fetch HTML", response.info(), 0) headers = response.info() kwds = {'referer': self._win.url} if headers.has_key('set-cookie'): kwds['cookie'] = headers['set-cookie'] if headers.has_key('last-modified'): kwds['lastModified'] = headers['last-modified'] return response.read(), kwds
def __init__(self, err): try: data = json.loads(err.fp.read().decode())['error'] except (ValueError, KeyError): # something REALLY bad happened and Facebook didn't send along # their usual error payload data = { 'message': 'Unhandled error', 'code': None, 'type': None, } HTTPError.__init__(self, err.url, err.code, data['message'], err.headers, err.fp) self.api_code = data['code'] self.type = data['type']
def __init__(self, err): self.logClassName = '.'.join([__name__, self.__class__.__name__]) self.logger = logging.getLogger(self.logClassName) try: data = simplejson.loads(err.fp.read().decode())['error'] self.logger.debug("Raw error response: %s", data) except (ValueError, KeyError): # something REALLY bad happened and Facebook didn't send along # their usual error payload data = { 'message': 'Unhandled error', 'code': None, 'type': None, } if '__debug__' in data: HTTPError.__init__(self, err.url, err.code, data['__debug__'], err.headers, err.fp) else: HTTPError.__init__(self, err.url, err.code, data['message'], err.headers, err.fp) self.api_code = data['code'] self.type = data['type']
def http_error_default(self, req, fp, code, msg, headers): """Provide an implementation for the default handler""" result = HTTPError( req.get_full_url(), code, msg, headers, fp) result.status = code return result
def http_error_default(self, req, fp, code, msg, headers): result = HTTPError(req.get_full_url(), code, msg, headers, fp) result.status = code return result