async def test_mandrill_send_client_error(cli, send_email, mocker): mock_mandrill_post = mocker.patch.object(cli.server.app['sender'].mandrill, 'post') mock_mandrill_post.side_effect = ClientError('foobar') es_url = 'messages/email-mandrill/_search?q=company:mandrill-error-test' r = await cli.server.app['es'].get(es_url, allowed_statuses='*') assert r.status == 200, await r.text() data = await r.json() assert data['hits']['total'] == 0 await send_email(method='email-mandrill', company_code='mandrill-error-test', recipients=[{ 'address': '*****@*****.**' }]) await cli.server.app['es'].get('messages/_refresh') r = await cli.server.app['es'].get(es_url, allowed_statuses='*') assert r.status == 200, await r.text() data = await r.json() assert data['hits']['total'] == 1 assert data['hits']['hits'][0]['_source'][ 'status'] == 'send_request_failed' assert data['hits']['hits'][0]['_source'][ 'body'] == 'Error sending email: ClientError'
async def check_anonymous(proxy: str) -> bool: """ 检测代理的匿名程度 :param proxy: 待校验的代理 :return: 校验结果,如果是高匿代理就返回True """ anonymous = True try: connector = ProxyConnector.from_url(proxy) requests.urllib3.disable_warnings() ua = UserAgent() async with ClientSession(connector=connector, timeout=5) as session: # 异步http请求 async with session.get(ANONYMOUS_CHECK_API, ssl=False, headers={"User-Agent": ua.random()}, timeout=5) as response: res = await response.text() res = json.loads(res) anonymous = ProxyValidator.is_anonymous(res) if anonymous: proxy_validator.info( "The proxy {} is anonymous".format(proxy)) await session.close() return anonymous except Exception as e: proxy_validator.error("Checking proxy {} anonymous " "has an error:{} type {}".format( proxy, str(e), type(e))) raise ClientError("check anonymous")
def test_monitor_site_aiohttp__no_message_error(event_loop): site = mock.MagicMock(url='http://example.com/', ok_http_status=302) with aioresponses() as mocked: mocked.get(site.url, exception=ClientError(None)) result, reason = event_loop.run_until_complete( monitor_site_aiohttp(site)) assert result == 'NG' assert reason == 'ClientError occurred:'
async def put_file( url: str, file_data: dict, extra_data: dict, *, retry: bool = True, max_attempts: int = 5, interval: float = 1.0, backoff: float = 0.25, request_timeout: float = 10.0, connector: BaseConnector = None, session: ClientSession = None, json: bool = False, ): """Put to HTTP server with automatic retries and timeouts. Args: url: the address to use file_data: dict with data key and path of file to upload extra_data: further content to include in data to put headers: an optional dict of headers to send retry: flag to retry the fetch max_attempts: the maximum number of attempts to make interval: the interval between retries, in seconds backoff: the backoff interval, in seconds request_timeout: the HTTP request timeout, in seconds connector: an optional existing BaseConnector session: a shared ClientSession json: flag to parse the result as JSON """ (data_key, file_path) = [k for k in file_data.items()][0] data = {**extra_data} limit = max_attempts if retry else 1 if not session: session = ClientSession(connector=connector, connector_owner=(not connector), trust_env=True) async with session: async for attempt in RepeatSequence(limit, interval, backoff): try: async with attempt.timeout(request_timeout): with open(file_path, "rb") as f: data[data_key] = f response: ClientResponse = await session.put(url, data=data) if (response.status < 200 or response.status >= 300 ) and (response.status != HTTPConflict.status_code): raise ClientError( f"Bad response from server: {response.status}, " f"{response.reason}") return await (response.json() if json else response.text()) except (ClientError, asyncio.TimeoutError) as e: if attempt.final: raise PutError("Exceeded maximum put attempts") from e
def test_error_logs_when_no_status_error(self): self.resp.error = ClientError('test error') with patch.object(self.logger, 'error') as error_mock,\ patch.object(self.logger, 'info') as info_mock: self.testobj.handle_response(self.resp) expected = '::error ::ClientError: test error' +\ ' - http://testing.test.com/' error_mock.assert_called_with(expected) info_mock.assert_not_called()
async def test_if_connection_error( self, m): exception = ClientError() self._prep_request(m, TEST_HOME_URL, exception=exception) async with ClientSession() as session: response = await self.testobj.fetch_response( session, self.urltarget) self.assertIs(self.urltarget, response.urltarget) self.assertEqual(0, response.status) self.assertIsNone(response.html) self.assertIs(exception, response.error) self.assertEqual(TEST_EXPECTED_ELAPSED, response.elapsed)
async def test_load_database_fallback_to_older_version_for_database_directory_if_is_update_required_failed( self): self.database.is_update_required = make_mocked_coro( raise_exception=ClientError()) self.database.current_version = "2018-04-15" self.database._is_database_present = MagicMock(return_value=True) with self.assertRaises(ClientError): await self.database.load_data("/path/to/database") self.assertEqual(self.database.database_directory, "/path/to/database/vane2_data_2018-04-15")
def test_perform_action_dont_start_scan_if_database_failed_to_download_and_no_older_database_present( self): self.vane.database.database_directory = None self.vane.database._load_data = make_mocked_coro( raise_exception=ClientError()) self.vane.scan_target = make_mocked_coro() with loop_context() as loop, patch("vane.core.custom_event_loop", MagicMock(return_value=loop)): self.vane.perform_action(action="scan", url="test", verify_ssl=False) self.vane.scan_target.assert_not_called()
async def test_async_validate_verification_code_with_invalid_code_response( self, mock_aioresponses): self._setup_session_response(mock_aioresponses, True, False) mock_aioresponses.post(API_VALIDATE_VERIFICATION_CODE_URLS["phone"], exception=ClientError()) authenticator = await self._async_create_authenticator_async( mock_aioresponses) await authenticator.async_authenticate() result = await authenticator.async_validate_verification_code("123456") self.assertEqual(ValidationResult.INVALID_VERIFICATION_CODE, result)
async def test_async_setup_raises_entry_not_ready(opp): """Test that it throws ConfigEntryNotReady when exception occurs during setup.""" config_entry = MockConfigEntry( domain=DOMAIN, data={CONF_URL: "https://some.url:1234"}, ) config_entry.add_to_opp(opp) with patch( "openpeerpower.components.nightscout.NightscoutAPI.get_server_status", side_effect=ClientError(), ): await opp.config_entries.async_setup(config_entry.entry_id) assert config_entry.state is ConfigEntryState.SETUP_RETRY
async def test_async_setup_raises_entry_not_ready(hass): """Test that it throws ConfigEntryNotReady when exception occurs during setup.""" config_entry = MockConfigEntry( domain=DOMAIN, data={CONF_URL: "https://some.url:1234"}, ) config_entry.add_to_hass(hass) with patch( "homeassistant.components.nightscout.NightscoutAPI.get_server_status", side_effect=ClientError(), ): await hass.config_entries.async_setup(config_entry.entry_id) assert config_entry.state == ENTRY_STATE_SETUP_RETRY
async def test_should_raises_client_error_exception( self, product_id, catalog_url ): with aioresponses() as m: m.get( catalog_url, exception=ClientError() ) with pytest.raises(CatalogException): await get_product_by_id( product_id='de83aecc-d490-4d96-8b0c-6c857a4736de' )
async def fetch( url: str, *, headers: dict = None, retry: bool = True, max_attempts: int = 5, interval: float = 1.0, backoff: float = 0.25, request_timeout: float = 10.0, connector: BaseConnector = None, session: ClientSession = None, json: bool = False, ): """Fetch from an HTTP server with automatic retries and timeouts. Args: url: the address to fetch headers: an optional dict of headers to send retry: flag to retry the fetch max_attempts: the maximum number of attempts to make interval: the interval between retries, in seconds backoff: the backoff interval, in seconds request_timeout: the HTTP request timeout, in seconds connector: an optional existing BaseConnector session: a shared ClientSession json: flag to parse the result as JSON """ limit = max_attempts if retry else 1 if not session: session = ClientSession(connector=connector, connector_owner=(not connector), trust_env=True) async with session: async for attempt in RepeatSequence(limit, interval, backoff): try: async with attempt.timeout(request_timeout): response: ClientResponse = await session.get( url, headers=headers) if response.status < 200 or response.status >= 300: raise ClientError( f"Bad response from server: {response.status} - " f"{response.reason}") return await (response.json() if json else response.text()) except (ClientError, asyncio.TimeoutError) as e: if attempt.final: raise FetchError("Exceeded maximum fetch attempts") from e
async def fetch_all_pages_url(session: ClientSession, run_in_executor: Callable) -> List[str]: logger.info('--- Fetching all urls... ---') async with session.get(settings.BASE_URL) as response: if response.status != 200: raise ClientError('Status code non equal 200') response_text = await response.text() soup = await run_in_executor(BeautifulSoup, response_text, 'lxml') tag = await run_in_executor(soup.find, settings.ARCHIVE_LIST_TAG, {'class': settings.ARCHIVE_LIST_SELECTOR}) result = [ url.attrs.get('href') for url in await run_in_executor(tag.findAll, 'a') ] logger.info(f'--- Found {len(result)} url(s) ---') return result
async def get_access_token(self) -> None: """ Get a Reddit API OAuth2 access token and assign it to self.access_token. A token is valid for 1 hour. There will be MAX_RETRIES to get a token, after which the cog will be unloaded and a ClientError raised if retrieval was still unsuccessful. """ for i in range(1, self.MAX_RETRIES + 1): response = await self.bot.http_session.post( url=f"{self.URL}/api/v1/access_token", headers=self.HEADERS, auth=self.client_auth, data={ "grant_type": "client_credentials", "duration": "temporary" }) if response.status == 200 and response.content_type == "application/json": content = await response.json() expiration = int(content["expires_in"] ) - 60 # Subtract 1 minute for leeway. self.access_token = AccessToken(token=content["access_token"], expires_at=datetime.utcnow() + timedelta(seconds=expiration)) log.debug( f"New token acquired; expires on UTC {self.access_token.expires_at}" ) return else: log.debug( f"Failed to get an access token: " f"status {response.status} & content type {response.content_type}; " f"retrying ({i}/{self.MAX_RETRIES})") await asyncio.sleep(3) self.bot.remove_cog(self.qualified_name) raise ClientError( "Authentication with the Reddit API failed. Unloading the cog.")
async def fetch_genesis_transactions(genesis_url: str): """Get genesis transactions.""" headers = {} headers["Content-Type"] = "application/json" retry = 5 LOGGER.info("Fetching genesis transactions from: %s", genesis_url) while True: try: async with ClientSession() as client_session: response: ClientResponse = await client_session.get( genesis_url, headers=headers) if response.status < 200 or response.status >= 300: raise ClientError("Bad response from server") genesis_txns = await response.text() return genesis_txns except ClientError as e: if retry: retry -= 1 await asyncio.sleep(5.0) else: raise ConfigError( "Error retrieving ledger genesis transactions") from e
async def cmd(self, command, params=None): """Send a command to the Blue Iris server. Parameters: command (str): Command to send to the Blue Iris server. Other Parameters: params (dict): Parameters for the command. (Default: None) Returns: dict: The 'data' portion of the JSON response. If there was no 'data' in the response, return the entire response JSON. """ if params is None: params = dict() args = { "session": self.blueiris_session, "response": self.response, "cmd": command } args.update(params) if self.debug: self.logger.info("Sending async command: {} {}".format( command, params)) self.logger.debug("Full command JSON data: {}".format(args)) try: async with self.async_websession.post( self.url, data=json.dumps(args)) as resp: rjson = await resp.json() if self.debug: self.logger.debug("Full json response: {}".format(rjson)) return rjson except ClientError as err: raise ClientError('Error requesting data from {}: {}'.format( self.url, err))
async def load_districts(self) -> Iterable: """Return a specific district.""" response = await self.session.get(url=f"{BASE_API_URL}/api/districts", allow_redirects=True) if response.status == 200: data = await response.json() results = [] last_update = data["lastUpdate"] for district in data["districts"]: name = district["name"] county = district["county"] count = district["count"] deaths = district["deaths"] week_incidence = round(district["weekIncidence"], 2) cases_per_100k = round(district["casesPer100k"], 2) cases_per_population = round(district["casesPerPopulation"], 2) results.append( DistrictData( name=name, county=county, count=count, deaths=deaths, weekIncidence=week_incidence, casesPer100k=cases_per_100k, casesPerPopulation=cases_per_population, lastUpdate=last_update, )) return results else: _LOGGER.error(f"Request failed {response}") raise ClientError(response)
async def error_task(product, version): raise ClientError('Error message')
) await hass.async_block_till_done() assert result2["type"] == "create_entry" assert result2["title"] == "*****@*****.**" assert result2["data"] == { "username": "******", "token": "test-token", } assert len(mock_setup.mock_calls) == 1 assert len(mock_setup_entry.mock_calls) == 1 @pytest.mark.parametrize( "error,reason", [(ClientError(), "cannot_connect"), (asyncio.TimeoutError(), "cannot_connect")], ) async def test_form_errors(hass, mock_login, mock_get_devices, error, reason): """Test we handle cannot connect error.""" mock_login.side_effect = error result = await hass.config_entries.flow.async_init( DOMAIN, context={"source": config_entries.SOURCE_USER}, data={ "username": "******", "password": "******" }, )
) await opp.async_block_till_done() assert result2["type"] == "create_entry" assert result2["title"] == "*****@*****.**" assert result2["data"] == { "username": "******", "token": "test-token", } assert len(mock_setup.mock_calls) == 1 assert len(mock_setup_entry.mock_calls) == 1 @pytest.mark.parametrize( "error,reason", [(ClientError(), "cannot_connect"), (asyncio.TimeoutError(), "cannot_connect")], ) async def test_form_errors(opp, mock_login, mock_get_devices, error, reason): """Test we handle cannot connect error.""" mock_login.side_effect = error result = await opp.config_entries.flow.async_init( DOMAIN, context={"source": config_entries.SOURCE_USER}, data={"username": "******", "password": "******"}, ) assert len(mock_login.mock_calls) == 1 assert result["type"] == "abort" assert result["reason"] == reason
def unreachable_store_in_ipt_anonymizer(m: aioresponses, url: str): m.post(url=url, exception=ClientError("Host unreachable"))
def test_baidu_connection_error_should_return_503(mocker): mock = mocker.patch('idiomfinder.main.scraper') mock.scrape_idioms.side_effect = ClientError() request, response = app.test_client.get('/?query=美丽') assert response.status == 503
async def test_unable_to_hit_next(self, app, message, mocker): """Test when unable to pass message to next service.""" mocker.patch('kafka_app.hit_next', side_effect=ClientError()) success = await app.process_message(message) assert not success
async def load_districts(self) -> Iterable: """Return a specific district.""" response = await self.session.get( url=f"{self.baseurl}{ENDPOINT_DISTRICTS}", allow_redirects=True ) if response.status == 200: data = await response.json() results = [] collectedData = {} last_update = data["meta"]["lastUpdate"] lastUpdate = datetime.strptime(last_update, "%Y-%m-%dT%H:%M:%S.%f%z") for dis in data["data"]: district = data["data"][dis] name = district["name"] county = district["county"] state = district["state"] population = district["population"] cases = district["cases"] deaths = district["deaths"] casesPerWeek = district["casesPerWeek"] recovered = district["recovered"] week_incidence = round(district["weekIncidence"], 2) cases_per_100k = round(district["casesPer100k"], 2) new_cases = district["delta"]["cases"] new_deaths = district["delta"]["deaths"] new_recovered = district["delta"]["recovered"] results.append( DistrictData( name=name, county=county, state=state, population=population, count=cases, deaths=deaths, casesPerWeek=casesPerWeek, recovered=recovered, weekIncidence=week_incidence, casesPer100k=cases_per_100k, newCases=new_cases, newDeaths=new_deaths, newRecovered=new_recovered, lastUpdate=lastUpdate, ) ) # Count data per state if state not in collectedData: collectedData[state] = DataCollector() collectedData[state].population = ( collectedData[state].population + population ) collectedData[state].count = collectedData[state].count + cases collectedData[state].deaths = collectedData[state].deaths + deaths collectedData[state].casesPerWeek = ( collectedData[state].casesPerWeek + casesPerWeek ) collectedData[state].recovered = ( collectedData[state].recovered + recovered ) # weekIncidence is calculated # casesPer100k is calculated collectedData[state].newCases = ( collectedData[state].newCases + new_cases ) collectedData[state].newDeaths = ( collectedData[state].newDeaths + new_deaths ) collectedData[state].newRecovered = ( collectedData[state].newRecovered + new_recovered ) collectedData[state].dataCount = collectedData[state].dataCount + 1 germanyTotal = DataCollector() for key, value in collectedData.items(): value.weekIncidence = round( value.casesPerWeek / value.population * 100000, 2 ) value.casesPer100k = round(value.count / value.population * 100000, 2) results.append( DistrictData( name="BL " + key, county="BL " + key, state="BL " + key, population=value.population, count=value.count, deaths=value.deaths, casesPerWeek=value.casesPerWeek, recovered=value.recovered, weekIncidence=value.weekIncidence, casesPer100k=value.casesPer100k, newCases=value.newCases, newDeaths=value.newDeaths, newRecovered=value.newRecovered, lastUpdate=lastUpdate, ) ) # Sum up the total values germanyTotal.population = germanyTotal.population + value.population germanyTotal.count = germanyTotal.count + value.count germanyTotal.deaths = germanyTotal.deaths + value.deaths germanyTotal.casesPerWeek = ( germanyTotal.casesPerWeek + value.casesPerWeek ) germanyTotal.recovered = germanyTotal.recovered + value.recovered # weekIncidence is calculated # casesPer100k is calculated germanyTotal.newCases = germanyTotal.newCases + value.newCases germanyTotal.newRecovered = ( germanyTotal.newRecovered + value.newRecovered ) germanyTotal.newDeaths = germanyTotal.newDeaths + value.newDeaths germanyTotal.dataCount = germanyTotal.dataCount + 1 germanyTotal.casesPer100k = round( germanyTotal.count / germanyTotal.population * 100000, 2 ) germanyTotal.weekIncidence = round( germanyTotal.casesPerWeek / germanyTotal.population * 100000, 2 ) results.append( DistrictData( name="Deutschland", county="Deutschland", state="", population=germanyTotal.population, count=germanyTotal.count, deaths=germanyTotal.deaths, casesPerWeek=germanyTotal.casesPerWeek, recovered=germanyTotal.recovered, weekIncidence=germanyTotal.weekIncidence, casesPer100k=germanyTotal.casesPer100k, newCases=germanyTotal.newCases, newDeaths=germanyTotal.newDeaths, newRecovered=germanyTotal.newRecovered, lastUpdate=lastUpdate, ) ) return results else: _LOGGER.error(f"Request failed {response}") raise ClientError(response)
TEST2_URL_PAGE2 = 'http://www.test2.com/page2.html' TEST2_HTML_PAGE2 = 'test2_page2' TEST2_URL_PAGE3 = 'http://www.test2.com/page3.html' TEST2_HTML_PAGE3 = 'test2_page3' TEST2_URL_FAVICON = 'http://www.test2.com/favicon.ico' TEST2_URL_LOGO = 'http://www.test2.com/logo.png' TEST3_URL_HOME = 'https://www.test3.com/' TEST3_HTML_HOME = 'test3_home' TEST3_URL_PAGE1 = 'https://www.test3.com/page1/index.html' TEST3_HTML_PAGE1 = 'test3_page1' TEST3_URL_PAGE2 = 'https://www.test3.com/page2/index.html' TEST3_HTML_PAGE2 = 'test3_page2' ERROR_404 = ClientResponseError(None, None, status=404) ERROR_CONNECTION = ClientError() # This class assists with mocking fetch responses, # can be reused multiple times to generate different # response instances, although the code should only # ever fetch a given url once class MockedResponseCreator: def __init__(self, html: str = None, error: Exception = None): self.html = html self.error = error def make_response(self, urltarget: UrlTarget): result = Mock(spec=UrlFetchResponse) result.urltarget = urltarget result.error = self.error