def testGetImageFromUrlSucceedsLastTry(self, mock_io, mock_requests): # Verify that we retry up to max retries for timeout errors. image_url = 'https://some.url.com/image.jpg' orig_img = Image.new('RGBA', (200, 200)) img_bytes = io.BytesIO() orig_img.save(img_bytes, format='png') mock_response = mock.MagicMock() mock_response.raw.read.return_value = 'something' mock_io.BytesIO.return_value = img_bytes mock_requests.get.side_effect = [ConnectTimeout(), ReadTimeout(), mock_response] output_img = atlasmaker_io.get_image(image_url, request_timeout=30, http_max_retries=3, http_retry_interval=0) self.assertEqual(mock_requests.get.call_count, 3) self.assertEqual(output_img.size, (200, 200))
def test_ncbi_fails(self): exceptions = [ChunkedEncodingError(), ConnectionError(), ReadTimeout(), ExpatError(), RuntimeError('bad record')] for code in [400, 429]: http_exception = HTTPError() http_exception.response = Response() http_exception.response.status_code = code exceptions.append(http_exception) for exception in exceptions: self.ncbi_exception = exception with self.assertLogs(level='DEBUG') as log: seq, tax = self.get_ncbi_data(query='MT345279.1') tax = tax.view(DataFrame) self.assertEqual( tax['Taxon']['MT345279.1'], 'k__Fungi; p__Basidiomycota; c__Agaricomycetes; ' 'o__Boletales; f__Boletaceae; g__Boletus; s__edulis' ) self.assertTrue('Retrying' in log.output[0])
def inner_wrapper(*args, **kwargs): final_excep = None for counter in range(times): if counter > 0: sleep(delay) try: value = function(*args, **kwargs) return value except ConnectionError as exception: if hasattr(exception.args[0], "reason") and isinstance( exception.args[0].reason, ReadTimeoutError): final_excep = ReadTimeout(exception.args[0]) else: raise exception except ReadTimeout as exception: final_excep = exception if final_excep is not None: raise final_excep
def test_last_block_not_recent_read_timeout(mock_time_range, mock_monero_rpc, caplog): """Raises a requests.exceptions.ReadTimeout""" mock_monero_rpc.side_effect = ReadTimeout( "Request timed out when reading response.") mock_time_range.return_value = (True, 12, "minutes") response = daemon_last_block_check() assert response["status"] == DAEMON_STATUS_UNKNOWN assert not response["block_recent"] assert response["block_recent_offset"] == 12 assert response["block_recent_offset_unit"] == "minutes" assert response["block_timestamp"] == "---" assert response["hash"] == "---" assert response["host"] == "127.0.0.1:18081" assert response["block_age"] == -1 assert "error" in response assert "error" in response["error"] assert "message" in response["error"] assert (response["error"]["error"] == "Request timed out when reading response.") assert (response["error"]["message"] == "Cannot determine status." ), "Wrong error." assert len(caplog.records) == 1 for record in caplog.records: assert record.levelname == "ERROR", "Wrong log message." json_message = json.loads(record.message) assert "message" in json_message, "Wrong log message." assert (json_message["message"] == "Cannot determine status." ), "Wrong log message." assert "error" in json_message, "Wrong log message." assert ( json_message["error"] == "Request timed out when reading response." ), "Wrong log message." caplog.clear()
def test_authentication_request_token_timeout(self): timeout = ReadTimeout('Read timed out. (read timeout=30)') responses.add( responses.POST, 'https://jira.example.com/plugins/servlet/oauth/request-token', body=timeout) # Start pipeline and go to setup page. self.client.get(self.setup_path) # Submit credentials data = { 'url': 'https://jira.example.com/', 'verify_ssl': False, 'consumer_key': 'sentry-bot', 'private_key': EXAMPLE_PRIVATE_KEY } resp = self.client.post(self.setup_path, data=data) assert resp.status_code == 200 self.assertContains(resp, 'Setup Error') self.assertContains(resp, 'request token from Jira') self.assertContains(resp, 'Timed out')
def test_site_monitor_handler(): mock = flexmock() mock.should_receive("total_seconds").and_return(0.1).and_return( 0.5).and_return(0.5).times(3) flexmock(boto3).should_receive("client").with_args("dynamodb").and_return( mock).twice() flexmock(index).should_receive("get_sites").with_args(mock).and_return( SITE_LIST).twice() bad_resp = flexmock(status_code=418, elapsed=mock) good_resp = flexmock(status_code=200, elapsed=mock) flexmock(requests).should_receive("get").with_args("https://mock.io", timeout=index.MAX_TIME)\ .and_return(bad_resp).and_raise(ReadTimeout()).twice() flexmock(requests).should_receive("get").with_args("https://mock.com", timeout=index.MAX_TIME)\ .and_return(good_resp).twice() flexmock(index).should_receive("save_stats").and_return(None).twice() flexmock(index).should_receive("send_alert").and_return(None).twice() fake_context = flexmock( invoked_function_arn="arn:aws:lambda:us-west-2:123456789:function:demo" ) assert index.site_monitor_handler(dict(), fake_context) is None assert index.site_monitor_handler(dict(), fake_context) is None
def request(self, method: str, form: web.Request, follow_redirects: bool = False, headers: dict = None) -> Page: """Submit the given form, returns a Page on success, None otherwise. @type method: str @type form: web.Request @type follow_redirects: bool @type headers: dict @rtype: Page """ form_headers = {} if isinstance(headers, dict) and len(headers): form_headers.update(headers) if form.referer: form_headers["referer"] = form.referer try: response = self._session.request(method, form.url, data=form.post_params, files=form.file_params, headers=form_headers, allow_redirects=follow_redirects, timeout=self._timeout, verify=self.secure) except ConnectionError as exception: # https://github.com/kennethreitz/requests/issues/2392 # Unfortunately chunked transfer + timeout raise ConnectionError... let's fix that if "Read timed out" in str(exception): raise ReadTimeout("Request time out") raise exception return Page(response)
def test_authentication_request_token_timeout(self): timeout = ReadTimeout("Read timed out. (read timeout=30)") responses.add( responses.POST, "https://jira.example.com/plugins/servlet/oauth/request-token", body=timeout, ) # Start pipeline and go to setup page. self.client.get(self.setup_path) # Submit credentials data = { "url": "https://jira.example.com/", "verify_ssl": False, "consumer_key": "sentry-bot", "private_key": EXAMPLE_PRIVATE_KEY, } resp = self.client.post(self.setup_path, data=data) assert resp.status_code == 200 self.assertContains(resp, "Setup Error") self.assertContains(resp, "request token from Jira") self.assertContains(resp, "Timed out")
def download_file(url, timeout, max_size, out=None): start = time.time() size = 0 r = requests.get(url, stream=True, timeout=timeout) r.raise_for_status() if int(r.headers.get('Content-Length', 0)) > max_size: raise HTTPError('Response too large.') out_file = out or BytesIO() for chunk in r.iter_content(1024): if time.time() - start > timeout: raise ReadTimeout('Timeout reached.') size += len(chunk) if size > max_size: raise HTTPError('Response too large.') out_file.write(chunk) if out is None: return out_file.getvalue() return out_file
def upload(filepath, url, auth_token, max_retries=5, timeout=300): basename = os.path.basename(filepath) click.echo( click.style('About to upload {} ({}) to {}'.format( filepath, sizeof_fmt(os.stat(filepath).st_size), url, ), fg='green')) retries = 0 sleeptime = 5 while True: try: t0 = time.time() response = requests.post( url, files={basename: open(filepath, 'rb')}, headers={ 'auth-token': auth_token, }, timeout=timeout, ) if response.status_code == 502: # force a re-attempt raise BadGatewayError(response.content) if response.status_code == 408 or response.status_code == 504: # Nginx calmly says Gunicorn timed out. Force a re-attempt. raise ReadTimeout(response.status_code) t1 = time.time() break except (ReadTimeout, BadGatewayError) as exception: t1 = time.time() retries += 1 click.echo( click.style( 'Deliberately sleeping for {} seconds'.format(sleeptime), fg='yellow')) time.sleep(sleeptime) if retries >= max_retries: raise click.echo( click.style('Retrying (after {:.1f}s) due to {}: {}'.format( t1 - t0, exception.__class__.__name__, exception, ), fg='yellow')) if response.status_code == 201: click.echo( click.style('Took {} seconds to upload {} ({} - {}/s)'.format( round(t1 - t0, 1), basename, sizeof_fmt(os.stat(filepath).st_size), sizeof_fmt(os.stat(filepath).st_size / (t1 - t0))), fg='green')) return True else: click.echo( click.style('Failed to upload! Status code: {}'.format( response.status_code))) try: click.echo(response.json()) except JSONDecodeError: click.echo(response.content) click.echo( click.style('Failed to upload! Status: {}'.format( response.status_code), fg='red'))
def run(_, command, **___): if 'file-with-error' in command: raise RequestConnectionError() if 'json-error' in command: return ContainerMock() raise ReadTimeout()
def json_callback(*args, **kwargs): raise ReadTimeout('too long')
def upload_by_download_url(url, auth_token, download_url, content_length=None, max_retries=5, timeout=300): click.echo( click.style('About to upload {} ({}) to {}'.format( download_url, 'n/a' if content_length is None else sizeof_fmt(content_length), url, ), fg='green')) retries = 0 sleeptime = 5 while True: try: t0 = time.time() response = requests.post( url, data={'url': download_url}, headers={ 'auth-token': auth_token, }, timeout=timeout, ) if response.status_code == 502: # force a re-attempt raise BadGatewayError(response.content) if response.status_code == 408 or response.status_code == 504: # Nginx calmly says Gunicorn timed out. Force a re-attempt. raise ReadTimeout(response.status_code) t1 = time.time() break except (ReadTimeout, BadGatewayError) as exception: t1 = time.time() retries += 1 click.echo( click.style( 'Deliberately sleeping for {} seconds'.format(sleeptime), fg='yellow')) time.sleep(sleeptime) if retries >= max_retries: raise click.echo( click.style('Retrying (after {:.1f}s) due to {}: {}'.format( t1 - t0, exception.__class__.__name__, exception, ), fg='yellow')) if response.status_code == 201: if content_length is None: click.echo( click.style( 'Took {} seconds to upload by download url {}'.format( round(t1 - t0, 1), download_url, ), fg='green')) else: click.echo( click.style('Took {} seconds to upload by download url ' '{} ({} - {}/s)'.format( round(t1 - t0, 1), download_url, sizeof_fmt(content_length), sizeof_fmt(content_length / (t1 - t0))), fg='green')) return True else: click.echo( click.style('Failed to upload! Status code: {}'.format( response.status_code))) try: click.echo(response.json()) except JSONDecodeError: click.echo(response.content) click.echo( click.style('Failed to upload! Status: {}'.format( response.status_code), fg='red'))
def request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=None, cert=None, **kwargs): """Constructs a :class:`Request <Request>`, prepares it and sends it. Returns :class:`Response <Response>` object. :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. :param auth: (optional) Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True by default. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. :param stream: (optional) whether to immediately download the response content. Defaults to ``False``. :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided. :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. """ #=============================================================================================================== # add by mz error_type = kwargs.get("error_type") if error_type: from requests.exceptions import InvalidURL, URLRequired, ConnectTimeout, ConnectionError, SSLError, ReadTimeout from requests.exceptions import InvalidSchema, MissingSchema, ChunkedEncodingError, ContentDecodingError from requests.exceptions import RequestException, HTTPError, ProxyError, Timeout, RetryError, StreamConsumedError get_error = { "InvalidURL": InvalidURL(), "URLRequired": URLRequired(), "ConnectTimeout": ConnectTimeout(), "ConnectionError": ConnectionError(), "SSLError": SSLError(), "ReadTimeout": ReadTimeout(), "InvalidSchema": InvalidSchema(), "MissingSchema": MissingSchema(), "ChunkedEncodingError": ChunkedEncodingError(), "ContentDecodingError": ContentDecodingError(), "StreamConsumedError": StreamConsumedError(), "TooManyRedirects": TooManyRedirects(), "RequestException": RequestException(), "HTTPError": HTTPError(), "ProxyError": ProxyError(), "Timeout": Timeout(), "RetryError": RetryError } error_ = get_error[error_type] raise error_ #=============================================================================================================== method = builtin_str(method) # Create the Request. req = Request( method=method.upper(), url=url, headers=headers, files=files, data=data or {}, params=params or {}, auth=auth, cookies=cookies, hooks=hooks, ) prep = self.prepare_request(req) proxies = proxies or {} # Gather clues from the surrounding environment. if self.trust_env: # Set environment's proxies. env_proxies = get_environ_proxies(url) or {} for (k, v) in env_proxies.items(): proxies.setdefault(k, v) # Look for configuration. if not verify and verify is not False: verify = os.environ.get('REQUESTS_CA_BUNDLE') # Curl compatibility. if not verify and verify is not False: verify = os.environ.get('CURL_CA_BUNDLE') # Merge all the kwargs. proxies = merge_setting(proxies, self.proxies) stream = merge_setting(stream, self.stream) verify = merge_setting(verify, self.verify) cert = merge_setting(cert, self.cert) # Send the request. send_kwargs = { 'stream': stream, 'timeout': timeout, 'verify': verify, 'cert': cert, 'proxies': proxies, 'allow_redirects': allow_redirects, } resp = self.send(prep, **send_kwargs) return resp
class OctoprintUploadAndStartJobTest(unittest.TestCase): def setUp(self): self.file_mock = tempfile.NamedTemporaryFile(delete=False) def tearDown(self): self.file_mock.close() os.remove(self.file_mock.name) @mock.patch("server.clients.octoprint.requests.Session.get") @mock.patch("server.clients.octoprint.requests.post") def test_upload_job_ok(self, mock_post_uri, mock_get_uri): mock_get_uri.return_value.status_code = 200 mock_get_uri.return_value.json.return_value = { "state": {"text": "Operational"}, "temperature": {}, } mock_post_uri.return_value.status_code = 201 mock_post_uri.return_value.json.return_value = { "files": { "local": { "name": "20mm-umlaut-box", "origin": "local", "refs": { "resource": "http://example.com/api/files/local/whistle_v2.gcode", "download": "http://example.com/downloads/files/local/whistle_v2.gcode", }, } }, "done": True, } printer = makeTestOctoprint( client_props={"connected": True}, ) printer.upload_and_start_job(self.file_mock.name) args, kwargs = mock_post_uri.call_args self.assertEqual(kwargs["data"], {"path": "karmen", "print": True}) self.assertEqual(kwargs["files"]["file"].name, self.file_mock.name) @mock.patch("server.clients.octoprint.requests.Session.get") @mock.patch("server.clients.octoprint.requests.post") def test_upload_job_path_ok(self, mock_post_uri, mock_get_uri): mock_get_uri.return_value.status_code = 200 mock_get_uri.return_value.json.return_value = { "state": {"text": "Operational"}, "temperature": {}, } mock_post_uri.return_value.status_code = 201 mock_post_uri.return_value.json.return_value = { "files": { "local": { "name": "20mm-umlaut-box", "origin": "local", "refs": { "resource": "http://example.com/api/files/local/whistle_v2.gcode", "download": "http://example.com/downloads/files/local/whistle_v2.gcode", }, } }, "done": True, } printer = makeTestOctoprint( client_props={"connected": True}, ) printer.upload_and_start_job( self.file_mock.name, path="sub/path/on/disk" ) kwargs = mock_post_uri.call_args[1] self.assertEqual( kwargs["data"], {"path": "karmen/sub/path/on/disk", "print": True} ) self.assertEqual(kwargs["files"]["file"].name, self.file_mock.name) @mock.patch("server.clients.octoprint.requests.Session.get") @mock.patch("server.clients.octoprint.requests.post", return_value=None) def test_upload_job_printing(self, mock_post_uri, mock_get_uri): mock_get_uri.return_value.status_code = 200 mock_get_uri.return_value.json.return_value = { "state": {"text": "Printing"}, "temperature": {}, } with self.assertRaises(DeviceInvalidState) as context: printer = makeTestOctoprint( client_props={"connected": True}, ) printer.upload_and_start_job(self.file_mock.name) @mock.patch("server.clients.octoprint.requests.Session.get") @mock.patch("server.clients.octoprint.requests.post", side_effect=ReadTimeout()) def test_upload_job_upload_crash(self, mock_post_uri, mock_get_uri): mock_get_uri.return_value.status_code = 200 mock_get_uri.return_value.json.return_value = { "state": {"text": "Operational"}, "temperature": {}, } printer = makeTestOctoprint( client_props={"connected": True}, ) with self.assertRaises(DeviceNetworkError): printer.upload_and_start_job(self.file_mock.name) @mock.patch("server.clients.octoprint.requests.Session.get") @mock.patch("server.clients.octoprint.requests.post") def test_upload_job_no_response(self, mock_post_uri, mock_get_uri): mock_get_uri.return_value.status_code = 200 mock_get_uri.return_value.json.return_value = { "state": {"text": "Operational"}, "temperature": {}, } printer = makeTestOctoprint( client_props={"connected": True}, ) with self.assertRaises(DeviceCommunicationError): printer.upload_and_start_job(self.file_mock.name)
def mocked_post(url, **kwargs): assert 'oauth/token' in url raise ReadTimeout('too long')
def exceptionCallback(request, uri, headers): raise ReadTimeout('Connection timed out.')
def _request(self, method, host, path, params=None, data=None, headers={}, response_type='json', max_retries=0, response_encoding=None, request_encoding=None, use_test_env=False, verify=False, cert=None, timeout=None, allow_non_200=False, files=None): """ Send a request to given destination :param str method: One of "GET/POST/HEAD/DELETE" :param str host: host, such as "http://www.qq.com/" :param str path: request path, like "/account/login/" :param str/dict params: params in query string :param str/dict data: data to send in POST request :param str response_type: type of response, can be one of "json" :param int max_retries: 最多可以重试的次数,默认为0,不重试 :param str response_encoding: 结果内容的编码,默认自动猜测 :param str request_encoding: 请求参数编码 :param str/bool verify: 是否校验crt :param string/tuple: 传递客户端crt和key :param int timtout: 超时时间 :returns: response """ url = self.make_url(host, path, use_test_env=use_test_env) request_exception = None resp, resp_status_code, resp_text = None, -1, '' result = None # 编码请求参数 params_to_send, data_to_send = params, data if request_encoding: if isinstance(params, dict): params_to_send = encode_dict(params, encoding=request_encoding) if isinstance(data, dict): data_to_send = encode_dict(data, encoding=request_encoding) try: client = self.smart_http_client logger.debug( 'Starting request to url=%s, params=%s, data=%s, headers=%s', url, params, data, json.dumps(headers)) resp = client.request(method, url, params=params_to_send, data=data_to_send, headers=headers, response_encoding=response_encoding, verify=verify, cert=cert, timeout=timeout, files=files) resp_text = resp['text'] resp_status_code = resp['status_code'] logger.debug( 'Response from url=%s, params=%s, data=%s, response=%s', url, params, data, resp_text) # Check response status if not allow_non_200 and resp_status_code != STATUS_CODE_OK: status_code = 'STATUS_CODE_%s' % resp_status_code raise Exception( 'Status Code: %s, Error Message: %s' % (resp_status_code, request_third_party_error_codes.error_codes.get( status_code, 'Third-party system interface is abnormal, %s' % resp['reason']))) except Exception as e: logger.exception('%s Error occured when sending request to %s', bk_error_codes.REQUEST_THIRD_PARTY_ERROR.code, url) if isinstance(e, ReadTimeout): request_exception = ReadTimeout( 'Third-party system interface response time exceeds %s seconds' % (timeout or REQUEST_TIMEOUT_SECS)) else: request_exception = e # 如果请求失败,而且max_retries > 0,尝试重试请求 if max_retries > 0: seconds_wait = 1 max_retries -= 1 # 在请求异常发生时,尝试漂移host地址 if isinstance(host, SmartHost): logger.info('Shift request host for %s', url) host.shift_host(use_test_env=use_test_env) logger.info( 'Will Retry request after %s seconds, remaining retries = %s', seconds_wait, max_retries) time.sleep(seconds_wait) return self._request(method, host, path, params, data, headers, response_type, max_retries, response_encoding, request_encoding, use_test_env) else: try: result = self.format_resp(resp_text, response_type=response_type) except Exception as e: logger.exception(u'%s resp_text: %s, response_type: %s', bk_error_codes.THIRD_PARTY_RESP_ERROR.code, resp_text, response_type) request_exception = e return FancyDict(url=url, resp=resp, resp_status_code=resp_status_code, resp_text=resp_text, result=result, request_exception=request_exception)
def run(_, arch_and_path, **___): if 'file-with-error' in arch_and_path: raise RequestConnectionError() if 'json-error' in arch_and_path: return ContainerMock() raise ReadTimeout()
def raise_read_timeout_error(reason, url): return MagicMock( side_effect=ReadTimeout(reason, request=MagicMock(url=url)))
class ImportChannelTestCase(TestCase): """ Test case for the importchannel management command. """ the_channel_id = '6199dde695db4ee4ab392222d5af1e5c' @patch('kolibri.content.management.commands.importchannel.paths.get_content_database_file_url') @patch('kolibri.content.management.commands.importchannel.paths.get_content_database_file_path') @patch('kolibri.content.management.commands.importchannel.transfer.FileDownload') @patch('kolibri.content.management.commands.importchannel.AsyncCommand.cancel', return_value=True) @patch('kolibri.content.management.commands.importchannel.AsyncCommand.is_cancelled', return_value=True) def test_remote_cancel_during_transfer(self, is_cancelled_mock, cancel_mock, FileDownloadMock, local_path_mock, remote_path_mock, start_progress_mock, import_channel_mock): local_path = tempfile.mkstemp()[1] local_path_mock.return_value = local_path remote_path_mock.return_value = 'notest' FileDownloadMock.return_value.__iter__.return_value = ['one', 'two', 'three'] call_command("importchannel", "network", self.the_channel_id) # Check that is_cancelled was called is_cancelled_mock.assert_called_with() # Check that the FileDownload initiated FileDownloadMock.assert_called_with('notest', local_path) # Check that cancel was called cancel_mock.assert_called_with() # Test that import channel cleans up database file if cancelled self.assertFalse(os.path.exists(local_path)) @patch('kolibri.content.management.commands.importchannel.paths.get_content_database_file_path') @patch('kolibri.content.management.commands.importchannel.transfer.FileCopy') @patch('kolibri.content.management.commands.importchannel.AsyncCommand.cancel', return_value=True) @patch('kolibri.content.management.commands.importchannel.AsyncCommand.is_cancelled', return_value=True) def test_local_cancel_during_transfer(self, is_cancelled_mock, cancel_mock, FileCopyMock, local_path_mock, start_progress_mock, import_channel_mock): local_dest_path = tempfile.mkstemp()[1] local_src_path = tempfile.mkstemp()[1] local_path_mock.side_effect = [local_dest_path, local_src_path] FileCopyMock.return_value.__iter__.return_value = ['one', 'two', 'three'] call_command("importchannel", "disk", self.the_channel_id, tempfile.mkdtemp()) # Check that is_cancelled was called is_cancelled_mock.assert_called_with() # Check that the FileCopy initiated FileCopyMock.assert_called_with(local_src_path, local_dest_path) # Check that cancel was called cancel_mock.assert_called_with() # Test that import channel cleans up database file if cancelled self.assertFalse(os.path.exists(local_dest_path)) @patch('kolibri.content.management.commands.importchannel.AsyncCommand.cancel') @patch('kolibri.content.management.commands.importchannel.AsyncCommand.is_cancelled', return_value=True) def test_remote_import_sslerror(self, is_cancelled_mock, cancel_mock, start_progress_mock, import_channel_mock): SSLERROR = SSLError(['SSL routines', 'ssl3_get_record', 'decryption failed or bad record mac']) if 'OpenSSL' in sys.modules: from OpenSSL.SSL import Error SSLERROR = Error(['SSL routines', 'ssl3_get_record', 'decryption failed or bad record mac']) with patch('kolibri.content.utils.transfer.Transfer.next', side_effect=SSLERROR): call_command('importchannel', 'network', '197934f144305350b5820c7c4dd8e194') cancel_mock.assert_called_with() import_channel_mock.assert_not_called() @patch('kolibri.content.utils.transfer.Transfer.next', side_effect=ReadTimeout('Read timed out.')) @patch('kolibri.content.management.commands.importchannel.AsyncCommand.cancel') @patch('kolibri.content.management.commands.importchannel.AsyncCommand.is_cancelled', return_value=True) def test_remote_import_readtimeout(self, is_cancelled_mock, cancel_mock, sslerror_mock, start_progress_mock, import_channel_mock): call_command('importchannel', 'network', '197934f144305350b5820c7c4dd8e194') cancel_mock.assert_called_with() import_channel_mock.assert_not_called()
def timed_out(id_token): raise ReadTimeout('too long')
def send(self, request, **kwargs): '''Sends a PreparedRequest object over FTP. Returns a response object. ''' # Get the authentication from the prepared request, if any. auth = self.get_username_password_from_header(request) # Next, get the host and the path. scheme, host, port, path = self.get_host_and_path_from_url(request) # Sort out the timeout. timeout = kwargs.get('timeout', None) # Look for a proxy proxies = kwargs.get('proxies', {}) proxy = proxies.get(scheme) # If there is a proxy, then we actually want to make a HTTP request if proxy: return self.send_proxy(request, proxy, **kwargs) # Establish the connection and login if needed. self.conn = ftplib.FTP() # Use a flag to distinguish read vs connection timeouts, and a flat set # of except blocks instead of a nested try-except, because python 3 # exception chaining makes things weird connected = False try: self.conn.connect(host, port, timeout) connected = True if auth is not None: self.conn.login(auth[0], auth[1]) else: self.conn.login() # Get the method and attempt to find the function to call. resp = self.func_table[request.method](path, request) except socket.timeout as e: # requests distinguishes between connection timeouts and others if connected: raise ReadTimeout(e, request=request) else: raise ConnectTimeout(e, request=request) # ftplib raises EOFError if the connection is unexpectedly closed. # Convert that or any other socket error to a ConnectionError. except (EOFError, socket.error) as e: raise ConnectionError(e, request=request) # Raised for 5xx errors. FTP uses 550 for both ENOENT and EPERM type # errors, so just translate all of these into a http-ish 404 except ftplib.error_perm as e: # The exception message is probably from the server, so if it's # non-ascii, who knows what the encoding is. Latin1 has the # advantage of not being able to fail. resp = build_text_response(request, BytesIO(str(e).encode('latin1')), str(codes.not_found)) # 4xx reply, translate to a http 503 except ftplib.error_temp as e: resp = build_text_response(request, BytesIO(str(e).encode('latin1')), str(codes.unavailable)) # error_reply is an unexpected status code, and error_proto is an # invalid status code. Error is the generic ftplib error, usually # raised when a line is too long. Translate all of them to a generic # RequestException except (ftplib.error_reply, ftplib.error_proto, ftplib.Error) as e: raise RequestException(e, request=request) # Return the response. return resp
def get(self, *args, **kw): raise ReadTimeout()
def test_raises_on_timeout_error(self): responses.add(responses.GET, "https://example.com", body=ReadTimeout()) session = Session("https://example.com", max_retries=1) with self.assertRaises(ReadTimeout): _ = session.get("")
def timeout_callback(http_request): if "sleep" in http_request.url: raise ReadTimeout("Read timed out") return 200, {}, "Hello there"
class ImportChannelTestCase(TestCase): """ Test case for the importchannel management command. """ the_channel_id = "6199dde695db4ee4ab392222d5af1e5c" @patch( "kolibri.core.content.management.commands.importchannel.paths.get_content_database_file_url" ) @patch( "kolibri.core.content.management.commands.importchannel.paths.get_content_database_file_path" ) @patch( "kolibri.core.content.management.commands.importchannel.transfer.FileDownload" ) @patch( "kolibri.core.content.management.commands.importchannel.AsyncCommand.cancel", return_value=True, ) @patch( "kolibri.core.content.management.commands.importchannel.AsyncCommand.is_cancelled", return_value=True, ) def test_remote_cancel_during_transfer( self, is_cancelled_mock, cancel_mock, FileDownloadMock, local_path_mock, remote_path_mock, start_progress_mock, import_channel_mock, ): local_path = tempfile.mkstemp()[1] local_path_mock.return_value = local_path remote_path_mock.return_value = "notest" FileDownloadMock.return_value.__iter__.return_value = ["one", "two", "three"] call_command("importchannel", "network", self.the_channel_id) # Check that is_cancelled was called is_cancelled_mock.assert_called_with() # Check that the FileDownload initiated FileDownloadMock.assert_called_with("notest", local_path) # Check that cancel was called cancel_mock.assert_called_with() # Test that import channel cleans up database file if cancelled self.assertFalse(os.path.exists(local_path)) @patch( "kolibri.core.content.management.commands.importchannel.paths.get_content_database_file_path" ) @patch("kolibri.core.content.management.commands.importchannel.transfer.FileCopy") @patch( "kolibri.core.content.management.commands.importchannel.AsyncCommand.cancel", return_value=True, ) @patch( "kolibri.core.content.management.commands.importchannel.AsyncCommand.is_cancelled", return_value=True, ) def test_local_cancel_during_transfer( self, is_cancelled_mock, cancel_mock, FileCopyMock, local_path_mock, start_progress_mock, import_channel_mock, ): local_dest_path = tempfile.mkstemp()[1] local_src_path = tempfile.mkstemp()[1] local_path_mock.side_effect = [local_dest_path, local_src_path] FileCopyMock.return_value.__iter__.return_value = ["one", "two", "three"] call_command("importchannel", "disk", self.the_channel_id, tempfile.mkdtemp()) # Check that is_cancelled was called is_cancelled_mock.assert_called_with() # Check that the FileCopy initiated FileCopyMock.assert_called_with(local_src_path, local_dest_path) # Check that cancel was called cancel_mock.assert_called_with() # Test that import channel cleans up database file if cancelled self.assertFalse(os.path.exists(local_dest_path)) @patch("kolibri.core.content.management.commands.importchannel.AsyncCommand.cancel") @patch( "kolibri.core.content.management.commands.importchannel.AsyncCommand.is_cancelled", return_value=True, ) def test_remote_import_sslerror( self, is_cancelled_mock, cancel_mock, start_progress_mock, import_channel_mock ): SSLERROR = SSLError( ["SSL routines", "ssl3_get_record", "decryption failed or bad record mac"] ) if "OpenSSL" in sys.modules: from OpenSSL.SSL import Error SSLERROR = Error( [ "SSL routines", "ssl3_get_record", "decryption failed or bad record mac", ] ) with patch( "kolibri.core.content.utils.transfer.Transfer.next", side_effect=SSLERROR ): call_command("importchannel", "network", "197934f144305350b5820c7c4dd8e194") cancel_mock.assert_called_with() import_channel_mock.assert_not_called() @patch( "kolibri.core.content.utils.transfer.Transfer.next", side_effect=ReadTimeout("Read timed out."), ) @patch("kolibri.core.content.management.commands.importchannel.AsyncCommand.cancel") @patch( "kolibri.core.content.management.commands.importchannel.AsyncCommand.is_cancelled", return_value=True, ) def test_remote_import_readtimeout( self, is_cancelled_mock, cancel_mock, sslerror_mock, start_progress_mock, import_channel_mock, ): call_command("importchannel", "network", "197934f144305350b5820c7c4dd8e194") cancel_mock.assert_called_with() import_channel_mock.assert_not_called()
def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None): """Follows `requests.HTTPAdapter.send()`, but does not send the last couple of bytes, instead storing them in `self._pending_requests`. """ # TODO: ensure no pooling happens here try: conn = self.get_connection(request.url, proxies) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError as e: # this may raise a string formatting error. err = ("Invalid timeout {}. Pass a (connect, read) " "timeout tuple, or a single float to set " "both timeouts to the same value".format(timeout)) raise ValueError(err) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) # this is what we will return for now. this object will later be # overwritten after the request is finished. response = Response() self.build_dummy_response_into(response, request) try: # Send the request. if hasattr(conn, 'proxy_pool'): conn = conn.proxy_pool low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT) try: low_conn.putrequest(request.method, url, skip_accept_encoding=True) for header, value in request.headers.items(): low_conn.putheader(header, value) if request.body is None: # no body # MASSIVE HACK ALERT: # at this point, *no* information has been sent to the # server---it is buffered and will only be sent after we # call low_conn.endheaders(). however, this request has no # body, so calling low_conn.endheaders() would actually # cause the request to be processed by the server # immediately. endheaders() is the only public way to get # the connection into a 'sent' state, which is the only # state in which you are allowed to read the response. so # we dig into the internals of the HTTPConnection object to # get it into the 'sent' state and send *most* of its # internal buffer without sending the two final newlines. # this follows HTTPConnection._send_output msg = b"\r\n".join(low_conn._buffer) del low_conn._buffer[:] low_conn.send(msg) low_conn._HTTPConnection__state = 'Request-sent' self._pending_requests.append( (request, low_conn, b'\r\n\r\n', response)) else: # some body, can end headers now low_conn.endheaders() body = request.body if isinstance(body, str): body = body.encode('utf-8') if 'Content-Length' in request.headers: # single body low_conn.send(body[:-3]) self._pending_requests.append( (request, low_conn, body[-3:], response)) else: # chunked body for i in request.body: low_conn.send(hex(len(i))[2:].encode('utf-8')) low_conn.send(b'\r\n') low_conn.send(i) low_conn.send(b'\r\n') self._pending_requests.append( (request, low_conn, b'0\r\n\r\n', response)) except: # If we hit any problems here, clean up the connection. # Then, reraise so that we can handle the actual exception. low_conn.close() raise except (ProtocolError, socket.error) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) raise ConnectionError(e, request=request) except ClosedPoolError as e: raise ConnectionError(e, request=request) except _ProxyError as e: raise ProxyError(e) except (_SSLError, _HTTPError) as e: if isinstance(e, _SSLError): # This branch is for urllib3 versions earlier than v1.22 raise SSLError(e, request=request) elif isinstance(e, ReadTimeoutError): raise ReadTimeout(e, request=request) else: raise return response
def mocked_get(url, **kwargs): assert '?access_token=' in url raise ReadTimeout('too long')
def request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=None, cert=None, json=None, **kwargs): """Constructs a :class:`Request <Request>`, prepares it and sends it. Returns :class:`Response <Response>` object. :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of ``'filename': file-like-objects`` for multipart encoding upload. :param auth: (optional) Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple :param allow_redirects: (optional) Set to True by default. :type allow_redirects: bool :param proxies: (optional) Dictionary mapping protocol or protocol and hostname to the URL of the proxy. :param stream: (optional) whether to immediately download the response content. Defaults to ``False``. :param verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``. :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. :rtype: requests.Response """ #=============================================================================================================== # add by mz error_type = kwargs.get("error_type") if error_type: from requests.exceptions import InvalidURL, URLRequired, ConnectTimeout, ConnectionError, SSLError, ReadTimeout from requests.exceptions import InvalidSchema, MissingSchema, ChunkedEncodingError, ContentDecodingError from requests.exceptions import RequestException, HTTPError, ProxyError, Timeout, RetryError, StreamConsumedError from requests.exceptions import TooManyRedirects get_error = { "InvalidURL": InvalidURL(), "URLRequired": URLRequired(), "ConnectTimeout": ConnectTimeout(), "ConnectionError": ConnectionError(), "SSLError": SSLError(), "ReadTimeout": ReadTimeout(), "InvalidSchema": InvalidSchema(), "MissingSchema": MissingSchema(), "ChunkedEncodingError": ChunkedEncodingError(), "ContentDecodingError": ContentDecodingError(), "StreamConsumedError": StreamConsumedError(), "TooManyRedirects": TooManyRedirects(), "RequestException": RequestException(), "HTTPError": HTTPError(), "ProxyError": ProxyError(), "Timeout": Timeout(), "RetryError": RetryError } error_ = get_error[error_type] raise error_ #=============================================================================================================== # Create the Request req = Request( method=method.upper(), url=url, headers=headers, files=files, data=data or {}, json=json, params=params or {}, auth=auth, cookies=cookies, hooks=hooks, ) prep = self.prepare_request(req) proxies = proxies or {} settings = self.merge_environment_settings(prep.url, proxies, stream, verify, cert) # Send the request. send_kwargs = { 'timeout': timeout, 'allow_redirects': allow_redirects, } send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) return resp