def test_close(self): conn = self.make_open_connector() session = ClientSession(loop=self.loop, connector=conn) session.close() self.assertIsNone(session.connector) self.assertTrue(conn.closed)
def test_init_cookies_with_list_of_tuples(self): session = ClientSession(cookies=[("c1", "cookie1"), ("c2", "cookie2")], loop=self.loop) self.assertEqual(set(session.cookies), {'c1', 'c2'}) self.assertEqual(session.cookies['c1'].value, 'cookie1') self.assertEqual(session.cookies['c2'].value, 'cookie2') session.close()
def test_init_headers_simple_dict(self): session = ClientSession(headers={ "h1": "header1", "h2": "header2" }, loop=self.loop) self.assertEqual(sorted(session._default_headers.items()), ([("H1", "header1"), ("H2", "header2")])) session.close()
def test_init_cookies_with_list_of_tuples(self): session = ClientSession( cookies=[("c1", "cookie1"), ("c2", "cookie2")], loop=self.loop) self.assertEqual(set(session.cookies), {'c1', 'c2'}) self.assertEqual(session.cookies['c1'].value, 'cookie1') self.assertEqual(session.cookies['c2'].value, 'cookie2') session.close()
def test_init_headers_list_of_tuples(self): session = ClientSession(headers=[("h1", "header1"), ("h2", "header2"), ("h3", "header3")], loop=self.loop) self.assertEqual( session._default_headers, CIMultiDict([("h1", "header1"), ("h2", "header2"), ("h3", "header3")])) session.close()
def test_init_cookies_with_simple_dict(self): session = ClientSession( cookies={ "c1": "cookie1", "c2": "cookie2" }, loop=self.loop) self.assertEqual(set(session.cookies), {'c1', 'c2'}) self.assertEqual(session.cookies['c1'].value, 'cookie1') self.assertEqual(session.cookies['c2'].value, 'cookie2') session.close()
def test_init_cookies_with_simple_dict(self): session = ClientSession(cookies={ "c1": "cookie1", "c2": "cookie2" }, loop=self.loop) self.assertEqual(set(session.cookies), {'c1', 'c2'}) self.assertEqual(session.cookies['c1'].value, 'cookie1') self.assertEqual(session.cookies['c2'].value, 'cookie2') session.close()
def test_init_headers_MultiDict(self): session = ClientSession(headers=MultiDict([("h1", "header1"), ("h2", "header2"), ("h3", "header3")]), loop=self.loop) self.assertEqual( session._default_headers, CIMultiDict([("H1", "header1"), ("H2", "header2"), ("H3", "header3")])) session.close()
def test_merge_headers_with_list_of_tuples(self): session = ClientSession(headers={ "h1": "header1", "h2": "header2" }, loop=self.loop) headers = session._prepare_headers([("h1", "h1")]) self.assertIsInstance(headers, CIMultiDict) self.assertEqual(headers, CIMultiDict([("h2", "header2"), ("h1", "h1")])) session.close()
def test_init_headers_simple_dict(self): session = ClientSession( headers={ "h1": "header1", "h2": "header2" }, loop=self.loop) self.assertEqual( sorted(session._default_headers.items()), ([("H1", "header1"), ("H2", "header2")])) session.close()
def test_merge_headers(self): # Check incoming simple dict session = ClientSession(headers={ "h1": "header1", "h2": "header2" }, loop=self.loop) headers = session._prepare_headers({"h1": "h1"}) self.assertIsInstance(headers, CIMultiDict) self.assertEqual(headers, CIMultiDict([("h2", "header2"), ("h1", "h1")])) session.close()
def test_init_headers_list_of_tuples(self): session = ClientSession( headers=[("h1", "header1"), ("h2", "header2"), ("h3", "header3")], loop=self.loop) self.assertEqual( session._default_headers, CIMultiDict([("h1", "header1"), ("h2", "header2"), ("h3", "header3")])) session.close()
def test_init_headers_MultiDict(self): session = ClientSession( headers=MultiDict( [("h1", "header1"), ("h2", "header2"), ("h3", "header3")]), loop=self.loop) self.assertEqual( session._default_headers, CIMultiDict([("H1", "header1"), ("H2", "header2"), ("H3", "header3")])) session.close()
def test_merge_headers_with_list_of_tuples(self): session = ClientSession( headers={ "h1": "header1", "h2": "header2" }, loop=self.loop) headers = session._prepare_headers([("h1", "h1")]) self.assertIsInstance(headers, CIMultiDict) self.assertEqual(headers, CIMultiDict([ ("h2", "header2"), ("h1", "h1") ])) session.close()
def test_merge_headers(self): # Check incoming simple dict session = ClientSession( headers={ "h1": "header1", "h2": "header2" }, loop=self.loop) headers = session._prepare_headers({ "h1": "h1" }) self.assertIsInstance(headers, CIMultiDict) self.assertEqual(headers, CIMultiDict([ ("h2", "header2"), ("h1", "h1") ])) session.close()
class AIOResponsesRaiseForStatusSessionTestCase(AsyncTestCase): """Test case for sessions with raise_for_status=True. This flag, introduced in aiohttp v2.0.0, automatically calls `raise_for_status()`. It is overridden by the `raise_for_status` argument of the request since aiohttp v3.4.a0. """ async def setup(self): self.url = 'http://example.com/api?foo=bar#fragment' self.session = ClientSession(raise_for_status=True) async def teardown(self): close_result = self.session.close() if close_result is not None: await close_result @aioresponses() async def test_raise_for_status(self, m): m.get(self.url, status=400) with self.assertRaises(ClientResponseError) as cm: await self.session.get(self.url) self.assertEqual(cm.exception.message, http.RESPONSES[400][0]) @aioresponses() @skipIf(condition=AIOHTTP_VERSION < '3.4.0', reason='aiohttp<3.4.0 does not support raise_for_status ' 'arguments for requests') async def test_do_not_raise_for_status(self, m): m.get(self.url, status=400) response = await self.session.get(self.url, raise_for_status=False) self.assertEqual(response.status, 400)
class AIOResponseRedirectTest(TestCase): @asyncio.coroutine def setUp(self): self.url = "http://10.1.1.1:8080/redirect" self.session = ClientSession() super().setUp() @asyncio.coroutine def tearDown(self): close_result = self.session.close() if close_result is not None: yield from close_result super().tearDown() @aioresponses() @asyncio.coroutine def test_redirect_followed(self, rsps): rsps.get( self.url, status=307, headers={"Location": "https://httpbin.org"}, ) rsps.get("https://httpbin.org") response = yield from self.session.get(self.url, allow_redirects=True) self.assertEqual(response.status, 200) self.assertEqual(str(response.url), "https://httpbin.org") self.assertEqual(len(response.history), 1) self.assertEqual(str(response.history[0].url), self.url) @aioresponses() @asyncio.coroutine def test_redirect_missing_mocked_match(self, rsps): rsps.get( self.url, status=307, headers={"Location": "https://httpbin.org"}, ) with self.assertRaises(ClientConnectionError) as cm: response = yield from self.session.get(self.url, allow_redirects=True) self.assertEqual( str(cm.exception), 'Connection refused: GET http://10.1.1.1:8080/redirect') @aioresponses() @asyncio.coroutine def test_redirect_missing_location_header(self, rsps): rsps.get(self.url, status=307) response = yield from self.session.get(self.url, allow_redirects=True) self.assertEqual(str(response.url), self.url)
class AIOResponsesTestCase(TestCase): use_default_loop = False @asyncio.coroutine def setUp(self): self.url = 'http://example.com/api?foo=bar#fragment' self.session = ClientSession() super().setUp() @asyncio.coroutine def tearDown(self): close_result = self.session.close() if close_result is not None: yield from close_result super().tearDown() def run_async(self, coroutine: Union[Coroutine, Generator]): return self.loop.run_until_complete(coroutine) @asyncio.coroutine def request(self, url: str): return (yield from self.session.get(url)) @data( hdrs.METH_HEAD, hdrs.METH_GET, hdrs.METH_POST, hdrs.METH_PUT, hdrs.METH_PATCH, hdrs.METH_DELETE, hdrs.METH_OPTIONS, ) @patch('aioresponses.aioresponses.add') @fail_on(unused_loop=False) def test_shortcut_method(self, http_method, mocked): with aioresponses() as m: getattr(m, http_method.lower())(self.url) mocked.assert_called_once_with(self.url, method=http_method) @aioresponses() def test_returned_instance(self, m): m.get(self.url) response = self.run_async(self.session.get(self.url)) self.assertIsInstance(response, ClientResponse) @aioresponses() @asyncio.coroutine def test_returned_instance_and_status_code(self, m): m.get(self.url, status=204) response = yield from self.session.get(self.url) self.assertIsInstance(response, ClientResponse) self.assertEqual(response.status, 204) @aioresponses() @asyncio.coroutine def test_returned_response_headers(self, m): m.get(self.url, content_type='text/html', headers={'Connection': 'keep-alive'}) response = yield from self.session.get(self.url) self.assertEqual(response.headers['Connection'], 'keep-alive') self.assertEqual(response.headers[hdrs.CONTENT_TYPE], 'text/html') @aioresponses() @asyncio.coroutine def test_returned_response_raw_headers(self, m): m.get(self.url, content_type='text/html', headers={'Connection': 'keep-alive'}) response = yield from self.session.get(self.url) expected_raw_headers = ((b'Content-Type', b'text/html'), (b'Connection', b'keep-alive')) self.assertEqual(response.raw_headers, expected_raw_headers) @aioresponses() @asyncio.coroutine def test_raise_for_status(self, m): m.get(self.url, status=400) with self.assertRaises(ClientResponseError) as cm: response = yield from self.session.get(self.url) response.raise_for_status() self.assertEqual(cm.exception.message, http.RESPONSES[400][0]) @aioresponses() @asyncio.coroutine def test_returned_instance_and_params_handling(self, m): expected_url = 'http://example.com/api?foo=bar&x=42#fragment' m.get(expected_url) response = yield from self.session.get(self.url, params={'x': 42}) self.assertIsInstance(response, ClientResponse) self.assertEqual(response.status, 200) expected_url = 'http://example.com/api?x=42#fragment' m.get(expected_url) response = yield from self.session.get( 'http://example.com/api#fragment', params={'x': 42}) self.assertIsInstance(response, ClientResponse) self.assertEqual(response.status, 200) @aioresponses() def test_method_dont_match(self, m): m.get(self.url) with self.assertRaises(ClientConnectionError): self.run_async(self.session.post(self.url)) @aioresponses() @asyncio.coroutine def test_streaming(self, m): m.get(self.url, body='Test') resp = yield from self.session.get(self.url) content = yield from resp.content.read() self.assertEqual(content, b'Test') @aioresponses() @asyncio.coroutine def test_streaming_up_to(self, m): m.get(self.url, body='Test') resp = yield from self.session.get(self.url) content = yield from resp.content.read(2) self.assertEqual(content, b'Te') content = yield from resp.content.read(2) self.assertEqual(content, b'st') @asyncio.coroutine def test_mocking_as_context_manager(self): with aioresponses() as aiomock: aiomock.add(self.url, payload={'foo': 'bar'}) resp = yield from self.session.get(self.url) self.assertEqual(resp.status, 200) payload = yield from resp.json() self.assertDictEqual(payload, {'foo': 'bar'}) def test_mocking_as_decorator(self): @aioresponses() def foo(loop, m): m.add(self.url, payload={'foo': 'bar'}) resp = loop.run_until_complete(self.session.get(self.url)) self.assertEqual(resp.status, 200) payload = loop.run_until_complete(resp.json()) self.assertDictEqual(payload, {'foo': 'bar'}) foo(self.loop) @asyncio.coroutine def test_passing_argument(self): @aioresponses(param='mocked') @asyncio.coroutine def foo(mocked): mocked.add(self.url, payload={'foo': 'bar'}) resp = yield from self.session.get(self.url) self.assertEqual(resp.status, 200) yield from foo() @fail_on(unused_loop=False) def test_mocking_as_decorator_wrong_mocked_arg_name(self): @aioresponses(param='foo') def foo(bar): # no matter what is here it should raise an error pass with self.assertRaises(TypeError) as cm: foo() exc = cm.exception self.assertIn("foo() got an unexpected keyword argument 'foo'", str(exc)) @asyncio.coroutine def test_unknown_request(self): with aioresponses() as aiomock: aiomock.add(self.url, payload={'foo': 'bar'}) with self.assertRaises(ClientConnectionError): yield from self.session.get('http://example.com/foo') @asyncio.coroutine def test_raising_custom_error(self): with aioresponses() as aiomock: aiomock.get(self.url, exception=HttpProcessingError(message='foo')) with self.assertRaises(HttpProcessingError): yield from self.session.get(self.url) @asyncio.coroutine def test_multiple_requests(self): with aioresponses() as m: m.get(self.url, status=200) m.get(self.url, status=201) m.get(self.url, status=202) resp = yield from self.session.get(self.url) self.assertEqual(resp.status, 200) resp = yield from self.session.get(self.url) self.assertEqual(resp.status, 201) resp = yield from self.session.get(self.url) self.assertEqual(resp.status, 202) key = ('GET', URL(self.url)) self.assertIn(key, m.requests) self.assertEqual(len(m.requests[key]), 3) self.assertEqual(m.requests[key][0].args, tuple()) self.assertEqual(m.requests[key][0].kwargs, {'allow_redirects': True}) @asyncio.coroutine def test_address_as_instance_of_url_combined_with_pass_through(self): external_api = 'http://httpbin.org/status/201' @asyncio.coroutine def doit(): api_resp = yield from self.session.get(self.url) # we have to hit actual url, # otherwise we do not test pass through option properly ext_rep = yield from self.session.get(URL(external_api)) return api_resp, ext_rep with aioresponses(passthrough=[external_api]) as m: m.get(self.url, status=200) api, ext = yield from doit() self.assertEqual(api.status, 200) self.assertEqual(ext.status, 201) @aioresponses() @asyncio.coroutine def test_custom_response_class(self, m): class CustomClientResponse(ClientResponse): pass m.get(self.url, body='Test', response_class=CustomClientResponse) resp = yield from self.session.get(self.url) self.assertTrue(isinstance(resp, CustomClientResponse)) @aioresponses() def test_exceptions_in_the_middle_of_responses(self, mocked): mocked.get(self.url, payload={}, status=204) mocked.get( self.url, exception=ValueError('oops'), ) mocked.get(self.url, payload={}, status=204) mocked.get( self.url, exception=ValueError('oops'), ) mocked.get(self.url, payload={}, status=200) @asyncio.coroutine def doit(): return (yield from self.session.get(self.url)) self.assertEqual(self.run_async(doit()).status, 204) with self.assertRaises(ValueError): self.run_async(doit()) self.assertEqual(self.run_async(doit()).status, 204) with self.assertRaises(ValueError): self.run_async(doit()) self.assertEqual(self.run_async(doit()).status, 200) @aioresponses() @asyncio.coroutine def test_request_should_match_regexp(self, mocked): mocked.get(re.compile(r'^http://example\.com/api\?foo=.*$'), payload={}, status=200) response = yield from self.request(self.url) self.assertEqual(response.status, 200) @aioresponses() @asyncio.coroutine def test_request_does_not_match_regexp(self, mocked): mocked.get(re.compile(r'^http://exampleexample\.com/api\?foo=.*$'), payload={}, status=200) with self.assertRaises(ClientConnectionError): yield from self.request(self.url) @aioresponses() def test_timeout(self, mocked): mocked.get(self.url, timeout=True) with self.assertRaises(asyncio.TimeoutError): self.run_async(self.request(self.url)) @aioresponses() def test_callback(self, m): body = b'New body' def callback(url, **kwargs): self.assertEqual(str(url), self.url) self.assertEqual(kwargs, {'allow_redirects': True}) return CallbackResult(body=body) m.get(self.url, callback=callback) response = self.run_async(self.request(self.url)) data = self.run_async(response.read()) assert data == body
def test_closed(self): session = ClientSession(loop=self.loop) self.assertFalse(session.closed) session.close() self.assertTrue(session.closed)
class ServiceClient: def __init__(self, rest_service_name='GenericService', spec=None, plugins=None, config=None, parser=None, serializer=None, base_path='', loop=None, logger=None): self._plugins = [] self.logger = logger or logging.getLogger('serviceClient.{}'.format(rest_service_name)) self.rest_service_name = rest_service_name self.spec = spec or {} self.add_plugins(plugins or []) self.config = config or {} self.parser = parser or (lambda x, *args, **kwargs: x) self.serializer = serializer or (lambda x, *args, **kwargs: x) self.base_path = base_path self.loop = loop or get_event_loop() self.connector = TCPConnector(loop=self.loop, **self.config.get('connector', {})) self.session = ClientSession(connector=self.connector, loop=self.loop) @coroutine def call(self, service_name, payload=None, **kwargs): self.logger.debug("Calling service_client {0}...".format(service_name)) service_desc = self.spec[service_name].copy() service_desc['service_name'] = service_name request_params = kwargs session = yield from self.prepare_session(service_desc, request_params) request_params['url'] = yield from self.generate_path(service_desc, session, request_params) request_params['method'] = service_desc.get('method', 'GET').upper() yield from self.prepare_request_params(service_desc, session, request_params) self.logger.info("Calling service_client {0} using {1} {2}".format(service_name, request_params['method'], request_params['url'])) payload = yield from self.prepare_payload(service_desc, session, request_params, payload) try: if request_params['method'] not in ['GET', 'DELETE']: try: stream_request = service_desc['stream_request'] except KeyError: stream_request = False if payload and not stream_request: request_params['data'] = self.serializer(payload, session=session, service_desc=service_desc, request_params=request_params) yield from self.before_request(service_desc, session, request_params) response = yield from session.request(**request_params) except Exception as e: self.logger.warn("Exception calling service_client {0}: {1}".format(service_name, e)) yield from self.on_exception(service_desc, session, request_params, e) raise e yield from self.on_response(service_desc, session, request_params, response) try: if service_desc['stream_response']: return response except KeyError: pass try: self.logger.info("Parsing response from {0}...".format(service_name)) response.data = self.parser((yield from response.read()), session=session, service_desc=service_desc, response=response) yield from self.on_parsed_response(service_desc, session, request_params, response) except Exception as e: self.logger.warn("[Response code: {0}] Exception parsing response from service_client " "{1}: {2}".format(response.status, service_name, e)) yield from self.on_parse_exception(service_desc, session, request_params, response, e) e.response = response raise e return response @coroutine def prepare_session(self, service_desc, request_params): session = SessionWrapper(self.session) yield from self._execute_plugin_hooks('prepare_session', service_desc=service_desc, session=session, request_params=request_params) return session @coroutine def generate_path(self, service_desc, session, request_params): path = service_desc.get('path', '') url = list(urlparse(self.base_path)) url[2] = '/'.join([url[2].rstrip('/'), path.lstrip('/')]) url.pop() path = urlunsplit(url) hooks = [getattr(plugin, 'prepare_path') for plugin in self._plugins if hasattr(plugin, 'prepare_path')] self.logger.debug("Calling {0} plugin hooks...".format('prepare_path')) for func in hooks: path = yield from func(service_desc=service_desc, session=session, request_params=request_params, path=path) return path @coroutine def prepare_request_params(self, service_desc, session, request_params): yield from self._execute_plugin_hooks('prepare_request_params', service_desc=service_desc, session=session, request_params=request_params) @coroutine def prepare_payload(self, service_desc, session, request_params, payload): hooks = [getattr(plugin, 'prepare_payload') for plugin in self._plugins if hasattr(plugin, 'prepare_payload')] self.logger.debug("Calling {0} plugin hooks...".format('prepare_payload')) for func in hooks: payload = yield from func(service_desc=service_desc, session=session, request_params=request_params, payload=payload) return payload @coroutine def before_request(self, service_desc, session, request_params): yield from self._execute_plugin_hooks('before_request', service_desc=service_desc, session=session, request_params=request_params) @coroutine def on_exception(self, service_desc, session, request_params, ex): yield from self._execute_plugin_hooks('on_exception', service_desc=service_desc, session=session, request_params=request_params, ex=ex) @coroutine def on_response(self, service_desc, session, request_params, response): yield from self._execute_plugin_hooks('on_response', service_desc=service_desc, session=session, request_params=request_params, response=response) @coroutine def on_parse_exception(self, service_desc, session, request_params, response, ex): yield from self._execute_plugin_hooks('on_parse_exception', service_desc=service_desc, session=session, request_params=request_params, response=response, ex=ex) @coroutine def on_parsed_response(self, service_desc, session, request_params, response): yield from self._execute_plugin_hooks('on_parsed_response', service_desc=service_desc, session=session, request_params=request_params, response=response) @coroutine def _execute_plugin_hooks(self, hook, *args, **kwargs): hooks = [getattr(plugin, hook) for plugin in self._plugins if hasattr(plugin, hook)] self.logger.debug("Calling {0} plugin hooks...".format(hook)) for func in hooks: yield from func(*args, **kwargs) def add_plugins(self, plugins): self._plugins.extend(plugins) hook = 'assign_service_client' hooks = [getattr(plugin, hook) for plugin in self._plugins if hasattr(plugin, hook)] self.logger.debug("Calling {0} plugin hooks...".format(hook)) for func in hooks: func(service_client=self) def __getattr__(self, item): @coroutine def wrap(*args, **kwargs): return self.call(item, *args, **kwargs) return wrap def __del__(self): # pragma: no cover self.session.close()
def go(): session = ClientSession(loop=self.loop) session.close() with self.assertRaises(RuntimeError): yield from session.request('get', '/')
def test_borrow_connector_loop(self): conn = self.make_open_connector() session = ClientSession(connector=conn) self.assertIs(session._loop, self.loop) session.close()
def function546(function647, function1293, arg1193): function1750 = ClientSession(connector=function647, loop=None) try: assert function1750._loop, loop finally: function1750.close()
class AIOResponsesTestCase(TestCase): use_default_loop = False @asyncio.coroutine def setUp(self): self.url = 'http://example.com/api' self.session = ClientSession() super().setUp() @asyncio.coroutine def tearDown(self): self.session.close() super().tearDown() @data( hdrs.METH_HEAD, hdrs.METH_GET, hdrs.METH_POST, hdrs.METH_PUT, hdrs.METH_PATCH, hdrs.METH_DELETE, hdrs.METH_OPTIONS, ) @patch('aioresponses.aioresponses.add') @fail_on(unused_loop=False) def test_shortcut_method(self, http_method, mocked): with aioresponses() as m: getattr(m, http_method.lower())(self.url) mocked.assert_called_once_with(self.url, method=http_method) @aioresponses() def test_returned_instance(self, m): m.get(self.url) response = self.loop.run_until_complete(self.session.get(self.url)) self.assertIsInstance(response, ClientResponse) @aioresponses() @asyncio.coroutine def test_returned_instance_and_status_code(self, m): m.get(self.url, status=204) response = yield from self.session.get(self.url) self.assertIsInstance(response, ClientResponse) self.assertEqual(response.status, 204) @aioresponses() @asyncio.coroutine def test_returned_response_headers(self, m): m.get(self.url, content_type='text/html', headers={'Connection': 'keep-alive'}) response = yield from self.session.get(self.url) self.assertEqual(response.headers['Connection'], 'keep-alive') self.assertEqual(response.headers[hdrs.CONTENT_TYPE], 'text/html') @aioresponses() def test_method_dont_match(self, m): m.get(self.url) with self.assertRaises(ClientConnectionError): self.loop.run_until_complete(self.session.post(self.url)) @aioresponses() @asyncio.coroutine def test_streaming(self, m): m.get(self.url, body='Test') resp = yield from self.session.get(self.url) content = yield from resp.content.read() self.assertEqual(content, b'Test') @aioresponses() @asyncio.coroutine def test_streaming_up_to(self, m): m.get(self.url, body='Test') resp = yield from self.session.get(self.url) content = yield from resp.content.read(2) self.assertEqual(content, b'Te') content = yield from resp.content.read(2) self.assertEqual(content, b'st') @asyncio.coroutine def test_mocking_as_context_manager(self): with aioresponses() as aiomock: aiomock.add(self.url, payload={'foo': 'bar'}) resp = yield from self.session.get(self.url) self.assertEqual(resp.status, 200) payload = yield from resp.json() self.assertDictEqual(payload, {'foo': 'bar'}) def test_mocking_as_decorator(self): @aioresponses() def foo(loop, m): m.add(self.url, payload={'foo': 'bar'}) resp = loop.run_until_complete(self.session.get(self.url)) self.assertEqual(resp.status, 200) payload = loop.run_until_complete(resp.json()) self.assertDictEqual(payload, {'foo': 'bar'}) foo(self.loop) @asyncio.coroutine def test_passing_argument(self): @aioresponses(param='mocked') @asyncio.coroutine def foo(mocked): mocked.add(self.url, payload={'foo': 'bar'}) resp = yield from self.session.get(self.url) self.assertEqual(resp.status, 200) yield from foo() @fail_on(unused_loop=False) def test_mocking_as_decorator_wrong_mocked_arg_name(self): @aioresponses(param='foo') def foo(bar): # no matter what is here it should raise an error pass with self.assertRaises(TypeError) as cm: foo() exc = cm.exception self.assertIn("foo() got an unexpected keyword argument 'foo'", str(exc)) @asyncio.coroutine def test_unknown_request(self): with aioresponses() as aiomock: aiomock.add(self.url, payload={'foo': 'bar'}) with self.assertRaises(ClientConnectionError): yield from self.session.get('http://example.com/foo') @asyncio.coroutine def test_raising_custom_error(self): with aioresponses() as aiomock: aiomock.get(self.url, exception=HttpProcessingError(message='foo')) with self.assertRaises(HttpProcessingError): yield from self.session.get(self.url) @asyncio.coroutine def test_multiple_requests(self): with aioresponses() as m: m.get(self.url, status=200) m.get(self.url, status=201) m.get(self.url, status=202) resp = yield from self.session.get(self.url) self.assertEqual(resp.status, 200) resp = yield from self.session.get(self.url) self.assertEqual(resp.status, 201) resp = yield from self.session.get(self.url) self.assertEqual(resp.status, 202) key = ('GET', self.url) self.assertIn(key, m.requests) self.assertEqual(len(m.requests[key]), 3) self.assertEqual(m.requests[key][0].args, tuple()) self.assertEqual(m.requests[key][0].kwargs, {'allow_redirects': True}) @asyncio.coroutine def test_address_as_instance_of_url_combined_with_pass_through(self): external_api = 'http://httpbin.org/status/201' @asyncio.coroutine def doit(): api_resp = yield from self.session.get(self.url) # we have to hit actual url, # otherwise we do not test pass through option properly ext_rep = yield from self.session.get(URL(external_api)) return api_resp, ext_rep with aioresponses(passthrough=[external_api]) as m: m.get(self.url, status=200) api, ext = yield from doit() self.assertEqual(api.status, 200) self.assertEqual(ext.status, 201)
def test_connector(self): connector = TCPConnector(loop=self.loop) session = ClientSession(connector=connector, loop=self.loop) self.assertIs(session.connector, connector) session.close()
class AIOResponseRedirectTest(AsyncTestCase): async def setup(self): self.url = "http://10.1.1.1:8080/redirect" self.session = ClientSession() async def teardown(self): close_result = self.session.close() if close_result is not None: await close_result @aioresponses() async def test_redirect_followed(self, rsps): rsps.get( self.url, status=307, headers={"Location": "https://httpbin.org"}, ) rsps.get("https://httpbin.org") response = await self.session.get(self.url, allow_redirects=True) self.assertEqual(response.status, 200) self.assertEqual(str(response.url), "https://httpbin.org") self.assertEqual(len(response.history), 1) self.assertEqual(str(response.history[0].url), self.url) @aioresponses() async def test_post_redirect_followed(self, rsps): rsps.post( self.url, status=307, headers={"Location": "https://httpbin.org"}, ) rsps.get("https://httpbin.org") response = await self.session.post(self.url, allow_redirects=True) self.assertEqual(response.status, 200) self.assertEqual(str(response.url), "https://httpbin.org") self.assertEqual(response.method, "get") self.assertEqual(len(response.history), 1) self.assertEqual(str(response.history[0].url), self.url) @aioresponses() async def test_redirect_missing_mocked_match(self, rsps): rsps.get( self.url, status=307, headers={"Location": "https://httpbin.org"}, ) with self.assertRaises(ClientConnectionError) as cm: await self.session.get(self.url, allow_redirects=True) self.assertEqual( str(cm.exception), 'Connection refused: GET http://10.1.1.1:8080/redirect') @aioresponses() async def test_redirect_missing_location_header(self, rsps): rsps.get(self.url, status=307) response = await self.session.get(self.url, allow_redirects=True) self.assertEqual(str(response.url), self.url) @aioresponses() @skipIf(condition=AIOHTTP_VERSION < '3.1.0', reason='aiohttp<3.1.0 does not add request info on response') async def test_request_info(self, rsps): rsps.get(self.url, status=200) response = await self.session.get(self.url) request_info = response.request_info assert str(request_info.url) == self.url assert request_info.headers == {} @aioresponses() @skipIf(condition=AIOHTTP_VERSION < '3.1.0', reason='aiohttp<3.1.0 does not add request info on response') async def test_request_info_with_original_request_headers(self, rsps): headers = {"Authorization": "Bearer access-token"} rsps.get(self.url, status=200) response = await self.session.get(self.url, headers=headers) request_info = response.request_info assert str(request_info.url) == self.url assert request_info.headers == headers
def test_cookies_are_readonly(self): session = ClientSession(loop=self.loop) with self.assertRaises(AttributeError): session.cookies = 123 session.close()
class AIOResponsesTestCase(AsyncTestCase): async def setup(self): self.url = 'http://example.com/api?foo=bar#fragment' self.session = ClientSession() async def teardown(self): close_result = self.session.close() if close_result is not None: await close_result def run_async(self, coroutine: Union[Coroutine, Generator]): return self.loop.run_until_complete(coroutine) async def request(self, url: str): return await self.session.get(url) @data( hdrs.METH_HEAD, hdrs.METH_GET, hdrs.METH_POST, hdrs.METH_PUT, hdrs.METH_PATCH, hdrs.METH_DELETE, hdrs.METH_OPTIONS, ) @patch('aioresponses.aioresponses.add') @fail_on(unused_loop=False) def test_shortcut_method(self, http_method, mocked): with aioresponses() as m: getattr(m, http_method.lower())(self.url) mocked.assert_called_once_with(self.url, method=http_method) @aioresponses() def test_returned_instance(self, m): m.get(self.url) response = self.run_async(self.session.get(self.url)) self.assertIsInstance(response, ClientResponse) @aioresponses() async def test_returned_instance_and_status_code(self, m): m.get(self.url, status=204) response = await self.session.get(self.url) self.assertIsInstance(response, ClientResponse) self.assertEqual(response.status, 204) @aioresponses() async def test_returned_response_headers(self, m): m.get(self.url, content_type='text/html', headers={'Connection': 'keep-alive'}) response = await self.session.get(self.url) self.assertEqual(response.headers['Connection'], 'keep-alive') self.assertEqual(response.headers[hdrs.CONTENT_TYPE], 'text/html') @aioresponses() async def test_returned_response_cookies(self, m): m.get(self.url, headers={'Set-Cookie': 'cookie=value'}) response = await self.session.get(self.url) self.assertEqual(response.cookies['cookie'].value, 'value') @aioresponses() async def test_returned_response_raw_headers(self, m): m.get(self.url, content_type='text/html', headers={'Connection': 'keep-alive'}) response = await self.session.get(self.url) expected_raw_headers = ((hdrs.CONTENT_TYPE.encode(), b'text/html'), (b'Connection', b'keep-alive')) self.assertEqual(response.raw_headers, expected_raw_headers) @aioresponses() async def test_raise_for_status(self, m): m.get(self.url, status=400) with self.assertRaises(ClientResponseError) as cm: response = await self.session.get(self.url) response.raise_for_status() self.assertEqual(cm.exception.message, http.RESPONSES[400][0]) @aioresponses() @skipIf(condition=AIOHTTP_VERSION < '3.4.0', reason='aiohttp<3.4.0 does not support raise_for_status ' 'arguments for requests') async def test_request_raise_for_status(self, m): m.get(self.url, status=400) with self.assertRaises(ClientResponseError) as cm: await self.session.get(self.url, raise_for_status=True) self.assertEqual(cm.exception.message, http.RESPONSES[400][0]) @aioresponses() async def test_returned_instance_and_params_handling(self, m): expected_url = 'http://example.com/api?foo=bar&x=42#fragment' m.get(expected_url) response = await self.session.get(self.url, params={'x': 42}) self.assertIsInstance(response, ClientResponse) self.assertEqual(response.status, 200) expected_url = 'http://example.com/api?x=42#fragment' m.get(expected_url) response = await self.session.get('http://example.com/api#fragment', params={'x': 42}) self.assertIsInstance(response, ClientResponse) self.assertEqual(response.status, 200) @aioresponses() def test_method_dont_match(self, m): m.get(self.url) with self.assertRaises(ClientConnectionError): self.run_async(self.session.post(self.url)) @aioresponses() async def test_streaming(self, m): m.get(self.url, body='Test') resp = await self.session.get(self.url) content = await resp.content.read() self.assertEqual(content, b'Test') @aioresponses() async def test_streaming_up_to(self, m): m.get(self.url, body='Test') resp = await self.session.get(self.url) content = await resp.content.read(2) self.assertEqual(content, b'Te') content = await resp.content.read(2) self.assertEqual(content, b'st') async def test_mocking_as_context_manager(self): with aioresponses() as aiomock: aiomock.add(self.url, payload={'foo': 'bar'}) resp = await self.session.get(self.url) self.assertEqual(resp.status, 200) payload = await resp.json() self.assertDictEqual(payload, {'foo': 'bar'}) def test_mocking_as_decorator(self): @aioresponses() def foo(loop, m): m.add(self.url, payload={'foo': 'bar'}) resp = loop.run_until_complete(self.session.get(self.url)) self.assertEqual(resp.status, 200) payload = loop.run_until_complete(resp.json()) self.assertDictEqual(payload, {'foo': 'bar'}) foo(self.loop) async def test_passing_argument(self): @aioresponses(param='mocked') async def foo(mocked): mocked.add(self.url, payload={'foo': 'bar'}) resp = await self.session.get(self.url) self.assertEqual(resp.status, 200) await foo() @fail_on(unused_loop=False) def test_mocking_as_decorator_wrong_mocked_arg_name(self): @aioresponses(param='foo') def foo(bar): # no matter what is here it should raise an error pass with self.assertRaises(TypeError) as cm: foo() exc = cm.exception self.assertIn("foo() got an unexpected keyword argument 'foo'", str(exc)) async def test_unknown_request(self): with aioresponses() as aiomock: aiomock.add(self.url, payload={'foo': 'bar'}) with self.assertRaises(ClientConnectionError): await self.session.get('http://example.com/foo') async def test_raising_exception(self): with aioresponses() as aiomock: url = 'http://example.com/Exception' aiomock.get(url, exception=Exception) with self.assertRaises(Exception): await self.session.get(url) url = 'http://example.com/Exception_object' aiomock.get(url, exception=Exception()) with self.assertRaises(Exception): await self.session.get(url) url = 'http://example.com/BaseException' aiomock.get(url, exception=BaseException) with self.assertRaises(BaseException): await self.session.get(url) url = 'http://example.com/BaseException_object' aiomock.get(url, exception=BaseException()) with self.assertRaises(BaseException): await self.session.get(url) url = 'http://example.com/CancelError' aiomock.get(url, exception=CancelledError) with self.assertRaises(CancelledError): await self.session.get(url) url = 'http://example.com/TimeoutError' aiomock.get(url, exception=TimeoutError) with self.assertRaises(TimeoutError): await self.session.get(url) url = 'http://example.com/HttpProcessingError' aiomock.get(url, exception=HttpProcessingError(message='foo')) with self.assertRaises(HttpProcessingError): await self.session.get(url) async def test_multiple_requests(self): """Ensure that requests are saved the way they would have been sent.""" with aioresponses() as m: m.get(self.url, status=200) m.get(self.url, status=201) m.get(self.url, status=202) json_content_as_ref = [1] resp = await self.session.get(self.url, json=json_content_as_ref) self.assertEqual(resp.status, 200) json_content_as_ref[:] = [2] resp = await self.session.get(self.url, json=json_content_as_ref) self.assertEqual(resp.status, 201) json_content_as_ref[:] = [3] resp = await self.session.get(self.url, json=json_content_as_ref) self.assertEqual(resp.status, 202) key = ('GET', URL(self.url)) self.assertIn(key, m.requests) self.assertEqual(len(m.requests[key]), 3) first_request = m.requests[key][0] self.assertEqual(first_request.args, tuple()) self.assertEqual(first_request.kwargs, { 'allow_redirects': True, "json": [1] }) second_request = m.requests[key][1] self.assertEqual(second_request.args, tuple()) self.assertEqual(second_request.kwargs, { 'allow_redirects': True, "json": [2] }) third_request = m.requests[key][2] self.assertEqual(third_request.args, tuple()) self.assertEqual(third_request.kwargs, { 'allow_redirects': True, "json": [3] }) async def test_request_with_non_deepcopyable_parameter(self): def non_deep_copyable(): """A generator does not allow deepcopy.""" for line in ["header1,header2", "v1,v2", "v10,v20"]: yield line generator_value = non_deep_copyable() with aioresponses() as m: m.get(self.url, status=200) resp = await self.session.get(self.url, data=generator_value) self.assertEqual(resp.status, 200) key = ('GET', URL(self.url)) self.assertIn(key, m.requests) self.assertEqual(len(m.requests[key]), 1) request = m.requests[key][0] self.assertEqual(request.args, tuple()) self.assertEqual(request.kwargs, { 'allow_redirects': True, "data": generator_value }) async def test_request_retrieval_in_case_no_response(self): with aioresponses() as m: with self.assertRaises(ClientConnectionError): await self.session.get(self.url) key = ('GET', URL(self.url)) self.assertIn(key, m.requests) self.assertEqual(len(m.requests[key]), 1) self.assertEqual(m.requests[key][0].args, tuple()) self.assertEqual(m.requests[key][0].kwargs, {'allow_redirects': True}) async def test_request_failure_in_case_session_is_closed(self): async def do_request(session): return (await session.get(self.url)) with aioresponses(): coro = do_request(self.session) await self.session.close() with self.assertRaises(RuntimeError) as exception_info: await coro assert str(exception_info.exception) == "Session is closed" async def test_address_as_instance_of_url_combined_with_pass_through(self): external_api = 'http://httpbin.org/status/201' async def doit(): api_resp = await self.session.get(self.url) # we have to hit actual url, # otherwise we do not test pass through option properly ext_rep = await self.session.get(URL(external_api)) return api_resp, ext_rep with aioresponses(passthrough=[external_api]) as m: m.get(self.url, status=200) api, ext = await doit() self.assertEqual(api.status, 200) self.assertEqual(ext.status, 201) async def test_pass_through_with_origin_params(self): external_api = 'http://httpbin.org/get' async def doit(params): # we have to hit actual url, # otherwise we do not test pass through option properly ext_rep = await self.session.get(URL(external_api), params=params) return ext_rep with aioresponses(passthrough=[external_api]) as m: params = {'foo': 'bar'} ext = await doit(params=params) self.assertEqual(ext.status, 200) self.assertEqual(str(ext.url), 'http://httpbin.org/get?foo=bar') @aioresponses() async def test_custom_response_class(self, m): class CustomClientResponse(ClientResponse): pass m.get(self.url, body='Test', response_class=CustomClientResponse) resp = await self.session.get(self.url) self.assertTrue(isinstance(resp, CustomClientResponse)) @aioresponses() def test_exceptions_in_the_middle_of_responses(self, mocked): mocked.get(self.url, payload={}, status=204) mocked.get( self.url, exception=ValueError('oops'), ) mocked.get(self.url, payload={}, status=204) mocked.get( self.url, exception=ValueError('oops'), ) mocked.get(self.url, payload={}, status=200) async def doit(): return (await self.session.get(self.url)) self.assertEqual(self.run_async(doit()).status, 204) with self.assertRaises(ValueError): self.run_async(doit()) self.assertEqual(self.run_async(doit()).status, 204) with self.assertRaises(ValueError): self.run_async(doit()) self.assertEqual(self.run_async(doit()).status, 200) @aioresponses() async def test_request_should_match_regexp(self, mocked): mocked.get(re.compile(r'^http://example\.com/api\?foo=.*$'), payload={}, status=200) response = await self.request(self.url) self.assertEqual(response.status, 200) @aioresponses() async def test_request_does_not_match_regexp(self, mocked): mocked.get(re.compile(r'^http://exampleexample\.com/api\?foo=.*$'), payload={}, status=200) with self.assertRaises(ClientConnectionError): await self.request(self.url) @aioresponses() def test_timeout(self, mocked): mocked.get(self.url, timeout=True) with self.assertRaises(asyncio.TimeoutError): self.run_async(self.request(self.url)) @aioresponses() def test_callback(self, m): body = b'New body' def callback(url, **kwargs): self.assertEqual(str(url), self.url) self.assertEqual(kwargs, {'allow_redirects': True}) return CallbackResult(body=body) m.get(self.url, callback=callback) response = self.run_async(self.request(self.url)) data = self.run_async(response.read()) assert data == body @aioresponses() def test_callback_coroutine(self, m): body = b'New body' event = asyncio.Event() async def callback(url, **kwargs): await event.wait() self.assertEqual(str(url), self.url) self.assertEqual(kwargs, {'allow_redirects': True}) return CallbackResult(body=body) m.get(self.url, callback=callback) future = asyncio.ensure_future(self.request(self.url)) self.run_async(asyncio.wait([future], timeout=0)) assert not future.done() event.set() self.run_async(asyncio.wait([future], timeout=0)) assert future.done() response = future.result() data = self.run_async(response.read()) assert data == body @aioresponses() async def test_exception_requests_are_tracked(self, mocked): kwargs = {"json": [42], "allow_redirects": True} mocked.get(self.url, exception=ValueError('oops')) with self.assertRaises(ValueError): await self.session.get(self.url, **kwargs) key = ('GET', URL(self.url)) mocked_requests = mocked.requests[key] self.assertEqual(len(mocked_requests), 1) request = mocked_requests[0] self.assertEqual(request.args, ()) self.assertEqual(request.kwargs, kwargs) async def test_possible_race_condition(self): async def random_sleep_cb(url, **kwargs): await asyncio.sleep(uniform(0.1, 1)) return CallbackResult(body='test') with aioresponses() as mocked: for i in range(20): mocked.get('http://example.org/id-{}'.format(i), callback=random_sleep_cb) tasks = [ self.session.get('http://example.org/id-{}'.format(i)) for i in range(20) ] await asyncio.gather(*tasks)
def test_create_session_outside_of_coroutine(loop): with pytest.warns(ResourceWarning): sess = ClientSession(loop=loop) sess.close()
class AIOResponsesTestCase(TestCase): def setUp(self): self.url = 'http://example.com/api' self.loop = asyncio.get_event_loop() self.session = ClientSession() def tearDown(self): self.session.close() @data( hdrs.METH_GET, hdrs.METH_POST, hdrs.METH_PUT, hdrs.METH_PATCH, hdrs.METH_DELETE, hdrs.METH_OPTIONS, ) @patch('aioresponses.aioresponses.add') def test_shortcut_method(self, http_method, mocked): with aioresponses() as m: getattr(m, http_method.lower())(self.url) mocked.assert_called_once_with(self.url, method=http_method) @aioresponses() def test_returned_instance(self, m): m.get(self.url) response = self.loop.run_until_complete(self.session.get(self.url)) self.assertIsInstance(response, ClientResponse) @aioresponses() def test_returned_response_headers(self, m): m.get(self.url, content_type='text/html', headers={'Connection': 'keep-alive'}) response = self.loop.run_until_complete(self.session.get(self.url)) self.assertEqual(response.headers['Connection'], 'keep-alive') self.assertEqual(response.headers[hdrs.CONTENT_TYPE], 'text/html') @aioresponses() def test_method_dont_match(self, m): m.get(self.url) with self.assertRaises(ClientConnectionError): self.loop.run_until_complete(self.session.post(self.url)) def test_mocking_as_context_manager(self): with aioresponses() as aiomock: aiomock.add(self.url, payload={'foo': 'bar'}) resp = self.loop.run_until_complete(self.session.get(self.url)) self.assertEqual(resp.status, 200) payload = self.loop.run_until_complete(resp.json()) self.assertDictEqual(payload, {'foo': 'bar'}) def test_mocking_as_decorator(self): @aioresponses() def foo(m): m.add(self.url, payload={'foo': 'bar'}) resp = self.loop.run_until_complete(self.session.get(self.url)) self.assertEqual(resp.status, 200) payload = self.loop.run_until_complete(resp.json()) self.assertDictEqual(payload, {'foo': 'bar'}) foo() def test_passing_argument(self): @aioresponses(param='mocked') def foo(mocked): mocked.add(self.url, payload={'foo': 'bar'}) resp = self.loop.run_until_complete(self.session.get(self.url)) self.assertEqual(resp.status, 200) foo() def test_mocking_as_decorator_wrong_mocked_arg_name(self): @aioresponses(param='foo') def foo(bar): # no matter what is here it should raise an error pass with self.assertRaises(TypeError) as cm: foo() exc = cm.exception self.assertIn("foo() got an unexpected keyword argument 'foo'", str(exc)) def test_unknown_request(self): with aioresponses() as aiomock: aiomock.add(self.url, payload={'foo': 'bar'}) with self.assertRaises(ClientConnectionError): self.loop.run_until_complete( self.session.get('http://example.com/foo')) def test_multiple_requests(self): with aioresponses() as m: m.get(self.url, status=200) m.get(self.url, status=201) m.get(self.url, status=202) resp = self.loop.run_until_complete(self.session.get(self.url)) self.assertEqual(resp.status, 200) resp = self.loop.run_until_complete(self.session.get(self.url)) self.assertEqual(resp.status, 201) resp = self.loop.run_until_complete(self.session.get(self.url)) self.assertEqual(resp.status, 202)
def test_close(self): session = ClientSession(loop=self.loop) session._connector = mock.Mock(BaseConnector) session.close() session._connector.close.assert_called_once_with()
def test_borrow_connector_loop(connector, create_session, loop): session = ClientSession(connector=connector, loop=None) try: assert session._loop, loop finally: session.close()
class LineStickerSpider: # default headers for requests headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36', } # url templates urlf_page = 'https://store.line.me/stickershop/product/{id}/en'.format urlf_search = 'https://store.line.me/api/search/sticker' \ '?query={query}&offset=0&limit=36&type=ALL&includeFacets=false'.format # sticker resolve order sticker_order = ['popup', 'animation', 'static'] re_sticker_id = re.compile('sticker/([^/]+)') # sticker id in url re_sticker_page_id = re.compile( 'stickershop/product/([^/]+)') # sticker shop id in url def __init__(self, connection_limit=10): connector = TCPConnector(limit=connection_limit) self.con = ClientSession(connector=connector, headers=self.headers) def __del__(self): asyncio.ensure_future(self.con.close()) async def crawl_search(self, query) -> Dict[str, str]: """ Crawls search and returns title-to-id dictionary, e.g.: {"cool sticker #1": '1234233', ...} """ url = self.urlf_search(query=query) response = await self.con.get(url) data = await response.json() results = {} for d in data['items']: results[d['title']] = d['id'] return results async def crawl_pages(self, page_ids: List[str]) -> Tuple[List[str], List[str]]: """Crawl sticker pages for sticker urls""" all_results = await gather(*[self.crawl_page(id_) for id_ in page_ids]) stickers = [] audio = [] for results in all_results: stickers.extend(results[0]) audio.extend(results[1]) return stickers, audio async def crawl_page(self, page_id): """crawl single sticker page and parse out sticker file urls""" response = await self.con.get(self.urlf_page(id=page_id)) return self.parse_page(await response.text()) async def dl_files(self, urls: List[str], output: Path) -> None: """Download files from urls asynchroniously""" return await gather(*[self.dl_file(url, output) for url in urls]) async def dl_file(self, url: str, output: Path) -> None: """Donwload file and save it to output""" id_ = self.re_sticker_id.findall(url)[0] response = await self.con.get(url) ext = os.path.splitext(urlparse(url).path)[1] with open(output / f'{id_}{ext}', 'wb') as f: f.write(await response.read()) def parse_page(self, body: str) -> Tuple[List[str], List[str]]: """Parse page for sticker urls""" sel = Selector(text=body) data = sel.css('.FnStickerPreviewItem::attr(data-preview)').extract() data = [json.loads(d) for d in data] stickers = [] audio = [] for d in data: sound = d.get(f'soundUrl') if sound: audio.append(sound) for key in self.sticker_order: value = d.get(f'{key}Url') if value: stickers.append(value) break return stickers, audio