async def test_mandrill_send_client_error(cli, send_email, mocker):
mock_mandrill_post = mocker.patch.object(cli.server.app['sender'].mandrill,
'post')
mock_mandrill_post.side_effect = ClientError('foobar')
es_url = 'messages/email-mandrill/_search?q=company:mandrill-error-test'
r = await cli.server.app['es'].get(es_url, allowed_statuses='*')
assert r.status == 200, await r.text()
data = await r.json()
assert data['hits']['total'] == 0
await send_email(method='email-mandrill',
company_code='mandrill-error-test',
recipients=[{
'address': '*****@*****.**'
}])
await cli.server.app['es'].get('messages/_refresh')
r = await cli.server.app['es'].get(es_url, allowed_statuses='*')
assert r.status == 200, await r.text()
data = await r.json()
assert data['hits']['total'] == 1
assert data['hits']['hits'][0]['_source'][
'status'] == 'send_request_failed'
assert data['hits']['hits'][0]['_source'][
'body'] == 'Error sending email: ClientError'
async def check_anonymous(proxy: str) -> bool:
"""
检测代理的匿名程度
:param proxy: 待校验的代理
:return: 校验结果,如果是高匿代理就返回True
"""
anonymous = True
try:
connector = ProxyConnector.from_url(proxy)
requests.urllib3.disable_warnings()
ua = UserAgent()
async with ClientSession(connector=connector,
timeout=5) as session:
# 异步http请求
async with session.get(ANONYMOUS_CHECK_API,
ssl=False,
headers={"User-Agent": ua.random()},
timeout=5) as response:
res = await response.text()
res = json.loads(res)
anonymous = ProxyValidator.is_anonymous(res)
if anonymous:
proxy_validator.info(
"The proxy {} is anonymous".format(proxy))
await session.close()
return anonymous
except Exception as e:
proxy_validator.error("Checking proxy {} anonymous "
"has an error:{} type {}".format(
proxy, str(e), type(e)))
raise ClientError("check anonymous")
def test_monitor_site_aiohttp__no_message_error(event_loop):
site = mock.MagicMock(url='http://example.com/', ok_http_status=302)
with aioresponses() as mocked:
mocked.get(site.url, exception=ClientError(None))
result, reason = event_loop.run_until_complete(
monitor_site_aiohttp(site))
assert result == 'NG'
assert reason == 'ClientError occurred:'
async def put_file(
url: str,
file_data: dict,
extra_data: dict,
*,
retry: bool = True,
max_attempts: int = 5,
interval: float = 1.0,
backoff: float = 0.25,
request_timeout: float = 10.0,
connector: BaseConnector = None,
session: ClientSession = None,
json: bool = False,
):
"""Put to HTTP server with automatic retries and timeouts.
Args:
url: the address to use
file_data: dict with data key and path of file to upload
extra_data: further content to include in data to put
headers: an optional dict of headers to send
retry: flag to retry the fetch
max_attempts: the maximum number of attempts to make
interval: the interval between retries, in seconds
backoff: the backoff interval, in seconds
request_timeout: the HTTP request timeout, in seconds
connector: an optional existing BaseConnector
session: a shared ClientSession
json: flag to parse the result as JSON
"""
(data_key, file_path) = [k for k in file_data.items()][0]
data = {**extra_data}
limit = max_attempts if retry else 1
if not session:
session = ClientSession(connector=connector,
connector_owner=(not connector),
trust_env=True)
async with session:
async for attempt in RepeatSequence(limit, interval, backoff):
try:
async with attempt.timeout(request_timeout):
with open(file_path, "rb") as f:
data[data_key] = f
response: ClientResponse = await session.put(url,
data=data)
if (response.status < 200 or response.status >= 300
) and (response.status !=
HTTPConflict.status_code):
raise ClientError(
f"Bad response from server: {response.status}, "
f"{response.reason}")
return await (response.json()
if json else response.text())
except (ClientError, asyncio.TimeoutError) as e:
if attempt.final:
raise PutError("Exceeded maximum put attempts") from e
def test_error_logs_when_no_status_error(self):
self.resp.error = ClientError('test error')
with patch.object(self.logger, 'error') as error_mock,\
patch.object(self.logger, 'info') as info_mock:
self.testobj.handle_response(self.resp)
expected = '::error ::ClientError: test error' +\
' - http://testing.test.com/'
error_mock.assert_called_with(expected)
info_mock.assert_not_called()
async def test_if_connection_error(
self, m):
exception = ClientError()
self._prep_request(m, TEST_HOME_URL, exception=exception)
async with ClientSession() as session:
response = await self.testobj.fetch_response(
session, self.urltarget)
self.assertIs(self.urltarget, response.urltarget)
self.assertEqual(0, response.status)
self.assertIsNone(response.html)
self.assertIs(exception, response.error)
self.assertEqual(TEST_EXPECTED_ELAPSED, response.elapsed)
async def test_load_database_fallback_to_older_version_for_database_directory_if_is_update_required_failed(
self):
self.database.is_update_required = make_mocked_coro(
raise_exception=ClientError())
self.database.current_version = "2018-04-15"
self.database._is_database_present = MagicMock(return_value=True)
with self.assertRaises(ClientError):
await self.database.load_data("/path/to/database")
self.assertEqual(self.database.database_directory,
"/path/to/database/vane2_data_2018-04-15")
def test_perform_action_dont_start_scan_if_database_failed_to_download_and_no_older_database_present(
self):
self.vane.database.database_directory = None
self.vane.database._load_data = make_mocked_coro(
raise_exception=ClientError())
self.vane.scan_target = make_mocked_coro()
with loop_context() as loop, patch("vane.core.custom_event_loop",
MagicMock(return_value=loop)):
self.vane.perform_action(action="scan",
url="test",
verify_ssl=False)
self.vane.scan_target.assert_not_called()
async def test_async_validate_verification_code_with_invalid_code_response(
self, mock_aioresponses):
self._setup_session_response(mock_aioresponses, True, False)
mock_aioresponses.post(API_VALIDATE_VERIFICATION_CODE_URLS["phone"],
exception=ClientError())
authenticator = await self._async_create_authenticator_async(
mock_aioresponses)
await authenticator.async_authenticate()
result = await authenticator.async_validate_verification_code("123456")
self.assertEqual(ValidationResult.INVALID_VERIFICATION_CODE, result)
async def test_async_setup_raises_entry_not_ready(opp):
"""Test that it throws ConfigEntryNotReady when exception occurs during setup."""
config_entry = MockConfigEntry(
domain=DOMAIN,
data={CONF_URL: "https://some.url:1234"},
)
config_entry.add_to_opp(opp)
with patch(
"openpeerpower.components.nightscout.NightscoutAPI.get_server_status",
side_effect=ClientError(),
):
await opp.config_entries.async_setup(config_entry.entry_id)
assert config_entry.state is ConfigEntryState.SETUP_RETRY
async def test_async_setup_raises_entry_not_ready(hass):
"""Test that it throws ConfigEntryNotReady when exception occurs during setup."""
config_entry = MockConfigEntry(
domain=DOMAIN,
data={CONF_URL: "https://some.url:1234"},
)
config_entry.add_to_hass(hass)
with patch(
"homeassistant.components.nightscout.NightscoutAPI.get_server_status",
side_effect=ClientError(),
):
await hass.config_entries.async_setup(config_entry.entry_id)
assert config_entry.state == ENTRY_STATE_SETUP_RETRY
async def test_should_raises_client_error_exception(
self,
product_id,
catalog_url
):
with aioresponses() as m:
m.get(
catalog_url,
exception=ClientError()
)
with pytest.raises(CatalogException):
await get_product_by_id(
product_id='de83aecc-d490-4d96-8b0c-6c857a4736de'
)
async def fetch(
url: str,
*,
headers: dict = None,
retry: bool = True,
max_attempts: int = 5,
interval: float = 1.0,
backoff: float = 0.25,
request_timeout: float = 10.0,
connector: BaseConnector = None,
session: ClientSession = None,
json: bool = False,
):
"""Fetch from an HTTP server with automatic retries and timeouts.
Args:
url: the address to fetch
headers: an optional dict of headers to send
retry: flag to retry the fetch
max_attempts: the maximum number of attempts to make
interval: the interval between retries, in seconds
backoff: the backoff interval, in seconds
request_timeout: the HTTP request timeout, in seconds
connector: an optional existing BaseConnector
session: a shared ClientSession
json: flag to parse the result as JSON
"""
limit = max_attempts if retry else 1
if not session:
session = ClientSession(connector=connector,
connector_owner=(not connector),
trust_env=True)
async with session:
async for attempt in RepeatSequence(limit, interval, backoff):
try:
async with attempt.timeout(request_timeout):
response: ClientResponse = await session.get(
url, headers=headers)
if response.status < 200 or response.status >= 300:
raise ClientError(
f"Bad response from server: {response.status} - "
f"{response.reason}")
return await (response.json() if json else response.text())
except (ClientError, asyncio.TimeoutError) as e:
if attempt.final:
raise FetchError("Exceeded maximum fetch attempts") from e
async def fetch_all_pages_url(session: ClientSession,
run_in_executor: Callable) -> List[str]:
logger.info('--- Fetching all urls... ---')
async with session.get(settings.BASE_URL) as response:
if response.status != 200:
raise ClientError('Status code non equal 200')
response_text = await response.text()
soup = await run_in_executor(BeautifulSoup, response_text, 'lxml')
tag = await run_in_executor(soup.find, settings.ARCHIVE_LIST_TAG,
{'class': settings.ARCHIVE_LIST_SELECTOR})
result = [
url.attrs.get('href')
for url in await run_in_executor(tag.findAll, 'a')
]
logger.info(f'--- Found {len(result)} url(s) ---')
return result
async def get_access_token(self) -> None:
"""
Get a Reddit API OAuth2 access token and assign it to self.access_token.
A token is valid for 1 hour. There will be MAX_RETRIES to get a token, after which the cog
will be unloaded and a ClientError raised if retrieval was still unsuccessful.
"""
for i in range(1, self.MAX_RETRIES + 1):
response = await self.bot.http_session.post(
url=f"{self.URL}/api/v1/access_token",
headers=self.HEADERS,
auth=self.client_auth,
data={
"grant_type": "client_credentials",
"duration": "temporary"
})
if response.status == 200 and response.content_type == "application/json":
content = await response.json()
expiration = int(content["expires_in"]
) - 60 # Subtract 1 minute for leeway.
self.access_token = AccessToken(token=content["access_token"],
expires_at=datetime.utcnow() +
timedelta(seconds=expiration))
log.debug(
f"New token acquired; expires on UTC {self.access_token.expires_at}"
)
return
else:
log.debug(
f"Failed to get an access token: "
f"status {response.status} & content type {response.content_type}; "
f"retrying ({i}/{self.MAX_RETRIES})")
await asyncio.sleep(3)
self.bot.remove_cog(self.qualified_name)
raise ClientError(
"Authentication with the Reddit API failed. Unloading the cog.")
async def fetch_genesis_transactions(genesis_url: str):
"""Get genesis transactions."""
headers = {}
headers["Content-Type"] = "application/json"
retry = 5
LOGGER.info("Fetching genesis transactions from: %s", genesis_url)
while True:
try:
async with ClientSession() as client_session:
response: ClientResponse = await client_session.get(
genesis_url, headers=headers)
if response.status < 200 or response.status >= 300:
raise ClientError("Bad response from server")
genesis_txns = await response.text()
return genesis_txns
except ClientError as e:
if retry:
retry -= 1
await asyncio.sleep(5.0)
else:
raise ConfigError(
"Error retrieving ledger genesis transactions") from e
async def cmd(self, command, params=None):
"""Send a command to the Blue Iris server.
Parameters:
command (str): Command to send to the Blue Iris server.
Other Parameters:
params (dict): Parameters for the command. (Default: None)
Returns:
dict: The 'data' portion of the JSON response. If there was no
'data' in the response, return the entire response JSON.
"""
if params is None:
params = dict()
args = {
"session": self.blueiris_session,
"response": self.response,
"cmd": command
}
args.update(params)
if self.debug:
self.logger.info("Sending async command: {} {}".format(
command, params))
self.logger.debug("Full command JSON data: {}".format(args))
try:
async with self.async_websession.post(
self.url, data=json.dumps(args)) as resp:
rjson = await resp.json()
if self.debug:
self.logger.debug("Full json response: {}".format(rjson))
return rjson
except ClientError as err:
raise ClientError('Error requesting data from {}: {}'.format(
self.url, err))
async def load_districts(self) -> Iterable:
"""Return a specific district."""
response = await self.session.get(url=f"{BASE_API_URL}/api/districts",
allow_redirects=True)
if response.status == 200:
data = await response.json()
results = []
last_update = data["lastUpdate"]
for district in data["districts"]:
name = district["name"]
county = district["county"]
count = district["count"]
deaths = district["deaths"]
week_incidence = round(district["weekIncidence"], 2)
cases_per_100k = round(district["casesPer100k"], 2)
cases_per_population = round(district["casesPerPopulation"], 2)
results.append(
DistrictData(
name=name,
county=county,
count=count,
deaths=deaths,
weekIncidence=week_incidence,
casesPer100k=cases_per_100k,
casesPerPopulation=cases_per_population,
lastUpdate=last_update,
))
return results
else:
_LOGGER.error(f"Request failed {response}")
raise ClientError(response)
async def error_task(product, version):
raise ClientError('Error message')
)
await hass.async_block_till_done()
assert result2["type"] == "create_entry"
assert result2["title"] == "*****@*****.**"
assert result2["data"] == {
"username": "******",
"token": "test-token",
}
assert len(mock_setup.mock_calls) == 1
assert len(mock_setup_entry.mock_calls) == 1
@pytest.mark.parametrize(
"error,reason",
[(ClientError(), "cannot_connect"),
(asyncio.TimeoutError(), "cannot_connect")],
)
async def test_form_errors(hass, mock_login, mock_get_devices, error, reason):
"""Test we handle cannot connect error."""
mock_login.side_effect = error
result = await hass.config_entries.flow.async_init(
DOMAIN,
context={"source": config_entries.SOURCE_USER},
data={
"username": "******",
"password": "******"
},
)
)
await opp.async_block_till_done()
assert result2["type"] == "create_entry"
assert result2["title"] == "*****@*****.**"
assert result2["data"] == {
"username": "******",
"token": "test-token",
}
assert len(mock_setup.mock_calls) == 1
assert len(mock_setup_entry.mock_calls) == 1
@pytest.mark.parametrize(
"error,reason",
[(ClientError(), "cannot_connect"), (asyncio.TimeoutError(), "cannot_connect")],
)
async def test_form_errors(opp, mock_login, mock_get_devices, error, reason):
"""Test we handle cannot connect error."""
mock_login.side_effect = error
result = await opp.config_entries.flow.async_init(
DOMAIN,
context={"source": config_entries.SOURCE_USER},
data={"username": "******", "password": "******"},
)
assert len(mock_login.mock_calls) == 1
assert result["type"] == "abort"
assert result["reason"] == reason
def unreachable_store_in_ipt_anonymizer(m: aioresponses, url: str):
m.post(url=url, exception=ClientError("Host unreachable"))
def test_baidu_connection_error_should_return_503(mocker):
mock = mocker.patch('idiomfinder.main.scraper')
mock.scrape_idioms.side_effect = ClientError()
request, response = app.test_client.get('/?query=美丽')
assert response.status == 503
async def test_unable_to_hit_next(self, app, message, mocker):
"""Test when unable to pass message to next service."""
mocker.patch('kafka_app.hit_next', side_effect=ClientError())
success = await app.process_message(message)
assert not success
async def load_districts(self) -> Iterable:
"""Return a specific district."""
response = await self.session.get(
url=f"{self.baseurl}{ENDPOINT_DISTRICTS}", allow_redirects=True
)
if response.status == 200:
data = await response.json()
results = []
collectedData = {}
last_update = data["meta"]["lastUpdate"]
lastUpdate = datetime.strptime(last_update, "%Y-%m-%dT%H:%M:%S.%f%z")
for dis in data["data"]:
district = data["data"][dis]
name = district["name"]
county = district["county"]
state = district["state"]
population = district["population"]
cases = district["cases"]
deaths = district["deaths"]
casesPerWeek = district["casesPerWeek"]
recovered = district["recovered"]
week_incidence = round(district["weekIncidence"], 2)
cases_per_100k = round(district["casesPer100k"], 2)
new_cases = district["delta"]["cases"]
new_deaths = district["delta"]["deaths"]
new_recovered = district["delta"]["recovered"]
results.append(
DistrictData(
name=name,
county=county,
state=state,
population=population,
count=cases,
deaths=deaths,
casesPerWeek=casesPerWeek,
recovered=recovered,
weekIncidence=week_incidence,
casesPer100k=cases_per_100k,
newCases=new_cases,
newDeaths=new_deaths,
newRecovered=new_recovered,
lastUpdate=lastUpdate,
)
)
# Count data per state
if state not in collectedData:
collectedData[state] = DataCollector()
collectedData[state].population = (
collectedData[state].population + population
)
collectedData[state].count = collectedData[state].count + cases
collectedData[state].deaths = collectedData[state].deaths + deaths
collectedData[state].casesPerWeek = (
collectedData[state].casesPerWeek + casesPerWeek
)
collectedData[state].recovered = (
collectedData[state].recovered + recovered
)
# weekIncidence is calculated
# casesPer100k is calculated
collectedData[state].newCases = (
collectedData[state].newCases + new_cases
)
collectedData[state].newDeaths = (
collectedData[state].newDeaths + new_deaths
)
collectedData[state].newRecovered = (
collectedData[state].newRecovered + new_recovered
)
collectedData[state].dataCount = collectedData[state].dataCount + 1
germanyTotal = DataCollector()
for key, value in collectedData.items():
value.weekIncidence = round(
value.casesPerWeek / value.population * 100000, 2
)
value.casesPer100k = round(value.count / value.population * 100000, 2)
results.append(
DistrictData(
name="BL " + key,
county="BL " + key,
state="BL " + key,
population=value.population,
count=value.count,
deaths=value.deaths,
casesPerWeek=value.casesPerWeek,
recovered=value.recovered,
weekIncidence=value.weekIncidence,
casesPer100k=value.casesPer100k,
newCases=value.newCases,
newDeaths=value.newDeaths,
newRecovered=value.newRecovered,
lastUpdate=lastUpdate,
)
)
# Sum up the total values
germanyTotal.population = germanyTotal.population + value.population
germanyTotal.count = germanyTotal.count + value.count
germanyTotal.deaths = germanyTotal.deaths + value.deaths
germanyTotal.casesPerWeek = (
germanyTotal.casesPerWeek + value.casesPerWeek
)
germanyTotal.recovered = germanyTotal.recovered + value.recovered
# weekIncidence is calculated
# casesPer100k is calculated
germanyTotal.newCases = germanyTotal.newCases + value.newCases
germanyTotal.newRecovered = (
germanyTotal.newRecovered + value.newRecovered
)
germanyTotal.newDeaths = germanyTotal.newDeaths + value.newDeaths
germanyTotal.dataCount = germanyTotal.dataCount + 1
germanyTotal.casesPer100k = round(
germanyTotal.count / germanyTotal.population * 100000, 2
)
germanyTotal.weekIncidence = round(
germanyTotal.casesPerWeek / germanyTotal.population * 100000, 2
)
results.append(
DistrictData(
name="Deutschland",
county="Deutschland",
state="",
population=germanyTotal.population,
count=germanyTotal.count,
deaths=germanyTotal.deaths,
casesPerWeek=germanyTotal.casesPerWeek,
recovered=germanyTotal.recovered,
weekIncidence=germanyTotal.weekIncidence,
casesPer100k=germanyTotal.casesPer100k,
newCases=germanyTotal.newCases,
newDeaths=germanyTotal.newDeaths,
newRecovered=germanyTotal.newRecovered,
lastUpdate=lastUpdate,
)
)
return results
else:
_LOGGER.error(f"Request failed {response}")
raise ClientError(response)
TEST2_URL_PAGE2 = 'http://www.test2.com/page2.html'
TEST2_HTML_PAGE2 = 'test2_page2'
TEST2_URL_PAGE3 = 'http://www.test2.com/page3.html'
TEST2_HTML_PAGE3 = 'test2_page3'
TEST2_URL_FAVICON = 'http://www.test2.com/favicon.ico'
TEST2_URL_LOGO = 'http://www.test2.com/logo.png'
TEST3_URL_HOME = 'https://www.test3.com/'
TEST3_HTML_HOME = 'test3_home'
TEST3_URL_PAGE1 = 'https://www.test3.com/page1/index.html'
TEST3_HTML_PAGE1 = 'test3_page1'
TEST3_URL_PAGE2 = 'https://www.test3.com/page2/index.html'
TEST3_HTML_PAGE2 = 'test3_page2'
ERROR_404 = ClientResponseError(None, None, status=404)
ERROR_CONNECTION = ClientError()
# This class assists with mocking fetch responses,
# can be reused multiple times to generate different
# response instances, although the code should only
# ever fetch a given url once
class MockedResponseCreator:
def __init__(self, html: str = None, error: Exception = None):
self.html = html
self.error = error
def make_response(self, urltarget: UrlTarget):
result = Mock(spec=UrlFetchResponse)
result.urltarget = urltarget
result.error = self.error
