def testGetImageFromUrlSucceedsLastTry(self, mock_io, mock_requests):
# Verify that we retry up to max retries for timeout errors.
image_url = 'https://some.url.com/image.jpg'
orig_img = Image.new('RGBA', (200, 200))
img_bytes = io.BytesIO()
orig_img.save(img_bytes, format='png')
mock_response = mock.MagicMock()
mock_response.raw.read.return_value = 'something'
mock_io.BytesIO.return_value = img_bytes
mock_requests.get.side_effect = [ConnectTimeout(), ReadTimeout(),
mock_response]
output_img = atlasmaker_io.get_image(image_url, request_timeout=30,
http_max_retries=3,
http_retry_interval=0)
self.assertEqual(mock_requests.get.call_count, 3)
self.assertEqual(output_img.size, (200, 200))
def test_ncbi_fails(self):
exceptions = [ChunkedEncodingError(), ConnectionError(), ReadTimeout(),
ExpatError(), RuntimeError('bad record')]
for code in [400, 429]:
http_exception = HTTPError()
http_exception.response = Response()
http_exception.response.status_code = code
exceptions.append(http_exception)
for exception in exceptions:
self.ncbi_exception = exception
with self.assertLogs(level='DEBUG') as log:
seq, tax = self.get_ncbi_data(query='MT345279.1')
tax = tax.view(DataFrame)
self.assertEqual(
tax['Taxon']['MT345279.1'],
'k__Fungi; p__Basidiomycota; c__Agaricomycetes; '
'o__Boletales; f__Boletaceae; g__Boletus; s__edulis'
)
self.assertTrue('Retrying' in log.output[0])
def inner_wrapper(*args, **kwargs):
final_excep = None
for counter in range(times):
if counter > 0:
sleep(delay)
try:
value = function(*args, **kwargs)
return value
except ConnectionError as exception:
if hasattr(exception.args[0], "reason") and isinstance(
exception.args[0].reason, ReadTimeoutError):
final_excep = ReadTimeout(exception.args[0])
else:
raise exception
except ReadTimeout as exception:
final_excep = exception
if final_excep is not None:
raise final_excep
def test_last_block_not_recent_read_timeout(mock_time_range, mock_monero_rpc,
caplog):
"""Raises a requests.exceptions.ReadTimeout"""
mock_monero_rpc.side_effect = ReadTimeout(
"Request timed out when reading response.")
mock_time_range.return_value = (True, 12, "minutes")
response = daemon_last_block_check()
assert response["status"] == DAEMON_STATUS_UNKNOWN
assert not response["block_recent"]
assert response["block_recent_offset"] == 12
assert response["block_recent_offset_unit"] == "minutes"
assert response["block_timestamp"] == "---"
assert response["hash"] == "---"
assert response["host"] == "127.0.0.1:18081"
assert response["block_age"] == -1
assert "error" in response
assert "error" in response["error"]
assert "message" in response["error"]
assert (response["error"]["error"] ==
"Request timed out when reading response.")
assert (response["error"]["message"] == "Cannot determine status."
), "Wrong error."
assert len(caplog.records) == 1
for record in caplog.records:
assert record.levelname == "ERROR", "Wrong log message."
json_message = json.loads(record.message)
assert "message" in json_message, "Wrong log message."
assert (json_message["message"] == "Cannot determine status."
), "Wrong log message."
assert "error" in json_message, "Wrong log message."
assert (
json_message["error"] == "Request timed out when reading response."
), "Wrong log message."
caplog.clear()
def test_authentication_request_token_timeout(self):
timeout = ReadTimeout('Read timed out. (read timeout=30)')
responses.add(
responses.POST,
'https://jira.example.com/plugins/servlet/oauth/request-token',
body=timeout)
# Start pipeline and go to setup page.
self.client.get(self.setup_path)
# Submit credentials
data = {
'url': 'https://jira.example.com/',
'verify_ssl': False,
'consumer_key': 'sentry-bot',
'private_key': EXAMPLE_PRIVATE_KEY
}
resp = self.client.post(self.setup_path, data=data)
assert resp.status_code == 200
self.assertContains(resp, 'Setup Error')
self.assertContains(resp, 'request token from Jira')
self.assertContains(resp, 'Timed out')
def test_site_monitor_handler():
mock = flexmock()
mock.should_receive("total_seconds").and_return(0.1).and_return(
0.5).and_return(0.5).times(3)
flexmock(boto3).should_receive("client").with_args("dynamodb").and_return(
mock).twice()
flexmock(index).should_receive("get_sites").with_args(mock).and_return(
SITE_LIST).twice()
bad_resp = flexmock(status_code=418, elapsed=mock)
good_resp = flexmock(status_code=200, elapsed=mock)
flexmock(requests).should_receive("get").with_args("https://mock.io", timeout=index.MAX_TIME)\
.and_return(bad_resp).and_raise(ReadTimeout()).twice()
flexmock(requests).should_receive("get").with_args("https://mock.com", timeout=index.MAX_TIME)\
.and_return(good_resp).twice()
flexmock(index).should_receive("save_stats").and_return(None).twice()
flexmock(index).should_receive("send_alert").and_return(None).twice()
fake_context = flexmock(
invoked_function_arn="arn:aws:lambda:us-west-2:123456789:function:demo"
)
assert index.site_monitor_handler(dict(), fake_context) is None
assert index.site_monitor_handler(dict(), fake_context) is None
def request(self,
method: str,
form: web.Request,
follow_redirects: bool = False,
headers: dict = None) -> Page:
"""Submit the given form, returns a Page on success, None otherwise.
@type method: str
@type form: web.Request
@type follow_redirects: bool
@type headers: dict
@rtype: Page
"""
form_headers = {}
if isinstance(headers, dict) and len(headers):
form_headers.update(headers)
if form.referer:
form_headers["referer"] = form.referer
try:
response = self._session.request(method,
form.url,
data=form.post_params,
files=form.file_params,
headers=form_headers,
allow_redirects=follow_redirects,
timeout=self._timeout,
verify=self.secure)
except ConnectionError as exception:
# https://github.com/kennethreitz/requests/issues/2392
# Unfortunately chunked transfer + timeout raise ConnectionError... let's fix that
if "Read timed out" in str(exception):
raise ReadTimeout("Request time out")
raise exception
return Page(response)
def test_authentication_request_token_timeout(self):
timeout = ReadTimeout("Read timed out. (read timeout=30)")
responses.add(
responses.POST,
"https://jira.example.com/plugins/servlet/oauth/request-token",
body=timeout,
)
# Start pipeline and go to setup page.
self.client.get(self.setup_path)
# Submit credentials
data = {
"url": "https://jira.example.com/",
"verify_ssl": False,
"consumer_key": "sentry-bot",
"private_key": EXAMPLE_PRIVATE_KEY,
}
resp = self.client.post(self.setup_path, data=data)
assert resp.status_code == 200
self.assertContains(resp, "Setup Error")
self.assertContains(resp, "request token from Jira")
self.assertContains(resp, "Timed out")
def download_file(url, timeout, max_size, out=None):
start = time.time()
size = 0
r = requests.get(url, stream=True, timeout=timeout)
r.raise_for_status()
if int(r.headers.get('Content-Length', 0)) > max_size:
raise HTTPError('Response too large.')
out_file = out or BytesIO()
for chunk in r.iter_content(1024):
if time.time() - start > timeout:
raise ReadTimeout('Timeout reached.')
size += len(chunk)
if size > max_size:
raise HTTPError('Response too large.')
out_file.write(chunk)
if out is None:
return out_file.getvalue()
return out_file
def upload(filepath, url, auth_token, max_retries=5, timeout=300):
basename = os.path.basename(filepath)
click.echo(
click.style('About to upload {} ({}) to {}'.format(
filepath,
sizeof_fmt(os.stat(filepath).st_size),
url,
),
fg='green'))
retries = 0
sleeptime = 5
while True:
try:
t0 = time.time()
response = requests.post(
url,
files={basename: open(filepath, 'rb')},
headers={
'auth-token': auth_token,
},
timeout=timeout,
)
if response.status_code == 502:
# force a re-attempt
raise BadGatewayError(response.content)
if response.status_code == 408 or response.status_code == 504:
# Nginx calmly says Gunicorn timed out. Force a re-attempt.
raise ReadTimeout(response.status_code)
t1 = time.time()
break
except (ReadTimeout, BadGatewayError) as exception:
t1 = time.time()
retries += 1
click.echo(
click.style(
'Deliberately sleeping for {} seconds'.format(sleeptime),
fg='yellow'))
time.sleep(sleeptime)
if retries >= max_retries:
raise
click.echo(
click.style('Retrying (after {:.1f}s) due to {}: {}'.format(
t1 - t0,
exception.__class__.__name__,
exception,
),
fg='yellow'))
if response.status_code == 201:
click.echo(
click.style('Took {} seconds to upload {} ({} - {}/s)'.format(
round(t1 - t0, 1), basename,
sizeof_fmt(os.stat(filepath).st_size),
sizeof_fmt(os.stat(filepath).st_size / (t1 - t0))),
fg='green'))
return True
else:
click.echo(
click.style('Failed to upload! Status code: {}'.format(
response.status_code)))
try:
click.echo(response.json())
except JSONDecodeError:
click.echo(response.content)
click.echo(
click.style('Failed to upload! Status: {}'.format(
response.status_code),
fg='red'))
def run(_, command, **___):
if 'file-with-error' in command:
raise RequestConnectionError()
if 'json-error' in command:
return ContainerMock()
raise ReadTimeout()
def json_callback(*args, **kwargs):
raise ReadTimeout('too long')
def upload_by_download_url(url,
auth_token,
download_url,
content_length=None,
max_retries=5,
timeout=300):
click.echo(
click.style('About to upload {} ({}) to {}'.format(
download_url,
'n/a' if content_length is None else sizeof_fmt(content_length),
url,
),
fg='green'))
retries = 0
sleeptime = 5
while True:
try:
t0 = time.time()
response = requests.post(
url,
data={'url': download_url},
headers={
'auth-token': auth_token,
},
timeout=timeout,
)
if response.status_code == 502:
# force a re-attempt
raise BadGatewayError(response.content)
if response.status_code == 408 or response.status_code == 504:
# Nginx calmly says Gunicorn timed out. Force a re-attempt.
raise ReadTimeout(response.status_code)
t1 = time.time()
break
except (ReadTimeout, BadGatewayError) as exception:
t1 = time.time()
retries += 1
click.echo(
click.style(
'Deliberately sleeping for {} seconds'.format(sleeptime),
fg='yellow'))
time.sleep(sleeptime)
if retries >= max_retries:
raise
click.echo(
click.style('Retrying (after {:.1f}s) due to {}: {}'.format(
t1 - t0,
exception.__class__.__name__,
exception,
),
fg='yellow'))
if response.status_code == 201:
if content_length is None:
click.echo(
click.style(
'Took {} seconds to upload by download url {}'.format(
round(t1 - t0, 1),
download_url,
),
fg='green'))
else:
click.echo(
click.style('Took {} seconds to upload by download url '
'{} ({} - {}/s)'.format(
round(t1 - t0, 1), download_url,
sizeof_fmt(content_length),
sizeof_fmt(content_length / (t1 - t0))),
fg='green'))
return True
else:
click.echo(
click.style('Failed to upload! Status code: {}'.format(
response.status_code)))
try:
click.echo(response.json())
except JSONDecodeError:
click.echo(response.content)
click.echo(
click.style('Failed to upload! Status: {}'.format(
response.status_code),
fg='red'))
def request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=None,
cert=None,
**kwargs):
"""Constructs a :class:`Request <Request>`, prepares it and sends it.
Returns :class:`Response <Response>` object.
:param method: method for the new :class:`Request` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary or bytes to be sent in the query
string for the :class:`Request`.
:param data: (optional) Dictionary or bytes to send in the body of the
:class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the
:class:`Request`.
:param cookies: (optional) Dict or CookieJar object to send with the
:class:`Request`.
:param files: (optional) Dictionary of 'filename': file-like-objects
for multipart encoding upload.
:param auth: (optional) Auth tuple or callable to enable
Basic/Digest/Custom HTTP Auth.
:param timeout: (optional) Float describing the timeout of the
request.
:param allow_redirects: (optional) Boolean. Set to True by default.
:param proxies: (optional) Dictionary mapping protocol to the URL of
the proxy.
:param stream: (optional) whether to immediately download the response
content. Defaults to ``False``.
:param verify: (optional) if ``True``, the SSL cert will be verified.
A CA_BUNDLE path can also be provided.
:param cert: (optional) if String, path to ssl client cert file (.pem).
If Tuple, ('cert', 'key') pair.
"""
#===============================================================================================================
# add by mz
error_type = kwargs.get("error_type")
if error_type:
from requests.exceptions import InvalidURL, URLRequired, ConnectTimeout, ConnectionError, SSLError, ReadTimeout
from requests.exceptions import InvalidSchema, MissingSchema, ChunkedEncodingError, ContentDecodingError
from requests.exceptions import RequestException, HTTPError, ProxyError, Timeout, RetryError, StreamConsumedError
get_error = {
"InvalidURL": InvalidURL(),
"URLRequired": URLRequired(),
"ConnectTimeout": ConnectTimeout(),
"ConnectionError": ConnectionError(),
"SSLError": SSLError(),
"ReadTimeout": ReadTimeout(),
"InvalidSchema": InvalidSchema(),
"MissingSchema": MissingSchema(),
"ChunkedEncodingError": ChunkedEncodingError(),
"ContentDecodingError": ContentDecodingError(),
"StreamConsumedError": StreamConsumedError(),
"TooManyRedirects": TooManyRedirects(),
"RequestException": RequestException(),
"HTTPError": HTTPError(),
"ProxyError": ProxyError(),
"Timeout": Timeout(),
"RetryError": RetryError
}
error_ = get_error[error_type]
raise error_
#===============================================================================================================
method = builtin_str(method)
# Create the Request.
req = Request(
method=method.upper(),
url=url,
headers=headers,
files=files,
data=data or {},
params=params or {},
auth=auth,
cookies=cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
proxies = proxies or {}
# Gather clues from the surrounding environment.
if self.trust_env:
# Set environment's proxies.
env_proxies = get_environ_proxies(url) or {}
for (k, v) in env_proxies.items():
proxies.setdefault(k, v)
# Look for configuration.
if not verify and verify is not False:
verify = os.environ.get('REQUESTS_CA_BUNDLE')
# Curl compatibility.
if not verify and verify is not False:
verify = os.environ.get('CURL_CA_BUNDLE')
# Merge all the kwargs.
proxies = merge_setting(proxies, self.proxies)
stream = merge_setting(stream, self.stream)
verify = merge_setting(verify, self.verify)
cert = merge_setting(cert, self.cert)
# Send the request.
send_kwargs = {
'stream': stream,
'timeout': timeout,
'verify': verify,
'cert': cert,
'proxies': proxies,
'allow_redirects': allow_redirects,
}
resp = self.send(prep, **send_kwargs)
return resp
class OctoprintUploadAndStartJobTest(unittest.TestCase):
def setUp(self):
self.file_mock = tempfile.NamedTemporaryFile(delete=False)
def tearDown(self):
self.file_mock.close()
os.remove(self.file_mock.name)
@mock.patch("server.clients.octoprint.requests.Session.get")
@mock.patch("server.clients.octoprint.requests.post")
def test_upload_job_ok(self, mock_post_uri, mock_get_uri):
mock_get_uri.return_value.status_code = 200
mock_get_uri.return_value.json.return_value = {
"state": {"text": "Operational"},
"temperature": {},
}
mock_post_uri.return_value.status_code = 201
mock_post_uri.return_value.json.return_value = {
"files": {
"local": {
"name": "20mm-umlaut-box",
"origin": "local",
"refs": {
"resource": "http://example.com/api/files/local/whistle_v2.gcode",
"download": "http://example.com/downloads/files/local/whistle_v2.gcode",
},
}
},
"done": True,
}
printer = makeTestOctoprint(
client_props={"connected": True},
)
printer.upload_and_start_job(self.file_mock.name)
args, kwargs = mock_post_uri.call_args
self.assertEqual(kwargs["data"], {"path": "karmen", "print": True})
self.assertEqual(kwargs["files"]["file"].name, self.file_mock.name)
@mock.patch("server.clients.octoprint.requests.Session.get")
@mock.patch("server.clients.octoprint.requests.post")
def test_upload_job_path_ok(self, mock_post_uri, mock_get_uri):
mock_get_uri.return_value.status_code = 200
mock_get_uri.return_value.json.return_value = {
"state": {"text": "Operational"},
"temperature": {},
}
mock_post_uri.return_value.status_code = 201
mock_post_uri.return_value.json.return_value = {
"files": {
"local": {
"name": "20mm-umlaut-box",
"origin": "local",
"refs": {
"resource": "http://example.com/api/files/local/whistle_v2.gcode",
"download": "http://example.com/downloads/files/local/whistle_v2.gcode",
},
}
},
"done": True,
}
printer = makeTestOctoprint(
client_props={"connected": True},
)
printer.upload_and_start_job(
self.file_mock.name, path="sub/path/on/disk"
)
kwargs = mock_post_uri.call_args[1]
self.assertEqual(
kwargs["data"], {"path": "karmen/sub/path/on/disk", "print": True}
)
self.assertEqual(kwargs["files"]["file"].name, self.file_mock.name)
@mock.patch("server.clients.octoprint.requests.Session.get")
@mock.patch("server.clients.octoprint.requests.post", return_value=None)
def test_upload_job_printing(self, mock_post_uri, mock_get_uri):
mock_get_uri.return_value.status_code = 200
mock_get_uri.return_value.json.return_value = {
"state": {"text": "Printing"},
"temperature": {},
}
with self.assertRaises(DeviceInvalidState) as context:
printer = makeTestOctoprint(
client_props={"connected": True},
)
printer.upload_and_start_job(self.file_mock.name)
@mock.patch("server.clients.octoprint.requests.Session.get")
@mock.patch("server.clients.octoprint.requests.post", side_effect=ReadTimeout())
def test_upload_job_upload_crash(self, mock_post_uri, mock_get_uri):
mock_get_uri.return_value.status_code = 200
mock_get_uri.return_value.json.return_value = {
"state": {"text": "Operational"},
"temperature": {},
}
printer = makeTestOctoprint(
client_props={"connected": True},
)
with self.assertRaises(DeviceNetworkError):
printer.upload_and_start_job(self.file_mock.name)
@mock.patch("server.clients.octoprint.requests.Session.get")
@mock.patch("server.clients.octoprint.requests.post")
def test_upload_job_no_response(self, mock_post_uri, mock_get_uri):
mock_get_uri.return_value.status_code = 200
mock_get_uri.return_value.json.return_value = {
"state": {"text": "Operational"},
"temperature": {},
}
printer = makeTestOctoprint(
client_props={"connected": True},
)
with self.assertRaises(DeviceCommunicationError):
printer.upload_and_start_job(self.file_mock.name)
def mocked_post(url, **kwargs):
assert 'oauth/token' in url
raise ReadTimeout('too long')
def exceptionCallback(request, uri, headers):
raise ReadTimeout('Connection timed out.')
def _request(self,
method,
host,
path,
params=None,
data=None,
headers={},
response_type='json',
max_retries=0,
response_encoding=None,
request_encoding=None,
use_test_env=False,
verify=False,
cert=None,
timeout=None,
allow_non_200=False,
files=None):
"""
Send a request to given destination
:param str method: One of "GET/POST/HEAD/DELETE"
:param str host: host, such as "http://www.qq.com/"
:param str path: request path, like "/account/login/"
:param str/dict params: params in query string
:param str/dict data: data to send in POST request
:param str response_type: type of response, can be one of "json"
:param int max_retries: 最多可以重试的次数，默认为0，不重试
:param str response_encoding: 结果内容的编码，默认自动猜测
:param str request_encoding: 请求参数编码
:param str/bool verify: 是否校验crt
:param string/tuple: 传递客户端crt和key
:param int timtout: 超时时间
:returns: response
"""
url = self.make_url(host, path, use_test_env=use_test_env)
request_exception = None
resp, resp_status_code, resp_text = None, -1, ''
result = None
# 编码请求参数
params_to_send, data_to_send = params, data
if request_encoding:
if isinstance(params, dict):
params_to_send = encode_dict(params, encoding=request_encoding)
if isinstance(data, dict):
data_to_send = encode_dict(data, encoding=request_encoding)
try:
client = self.smart_http_client
logger.debug(
'Starting request to url=%s, params=%s, data=%s, headers=%s',
url, params, data, json.dumps(headers))
resp = client.request(method,
url,
params=params_to_send,
data=data_to_send,
headers=headers,
response_encoding=response_encoding,
verify=verify,
cert=cert,
timeout=timeout,
files=files)
resp_text = resp['text']
resp_status_code = resp['status_code']
logger.debug(
'Response from url=%s, params=%s, data=%s, response=%s', url,
params, data, resp_text)
# Check response status
if not allow_non_200 and resp_status_code != STATUS_CODE_OK:
status_code = 'STATUS_CODE_%s' % resp_status_code
raise Exception(
'Status Code: %s, Error Message: %s' %
(resp_status_code,
request_third_party_error_codes.error_codes.get(
status_code,
'Third-party system interface is abnormal, %s' %
resp['reason'])))
except Exception as e:
logger.exception('%s Error occured when sending request to %s',
bk_error_codes.REQUEST_THIRD_PARTY_ERROR.code,
url)
if isinstance(e, ReadTimeout):
request_exception = ReadTimeout(
'Third-party system interface response time exceeds %s seconds'
% (timeout or REQUEST_TIMEOUT_SECS))
else:
request_exception = e
# 如果请求失败，而且max_retries > 0，尝试重试请求
if max_retries > 0:
seconds_wait = 1
max_retries -= 1
# 在请求异常发生时，尝试漂移host地址
if isinstance(host, SmartHost):
logger.info('Shift request host for %s', url)
host.shift_host(use_test_env=use_test_env)
logger.info(
'Will Retry request after %s seconds, remaining retries = %s',
seconds_wait, max_retries)
time.sleep(seconds_wait)
return self._request(method, host, path, params, data, headers,
response_type, max_retries,
response_encoding, request_encoding,
use_test_env)
else:
try:
result = self.format_resp(resp_text,
response_type=response_type)
except Exception as e:
logger.exception(u'%s resp_text: %s, response_type: %s',
bk_error_codes.THIRD_PARTY_RESP_ERROR.code,
resp_text, response_type)
request_exception = e
return FancyDict(url=url,
resp=resp,
resp_status_code=resp_status_code,
resp_text=resp_text,
result=result,
request_exception=request_exception)
def run(_, arch_and_path, **___):
if 'file-with-error' in arch_and_path:
raise RequestConnectionError()
if 'json-error' in arch_and_path:
return ContainerMock()
raise ReadTimeout()
def raise_read_timeout_error(reason, url):
return MagicMock(
side_effect=ReadTimeout(reason, request=MagicMock(url=url)))
class ImportChannelTestCase(TestCase):
"""
Test case for the importchannel management command.
"""
the_channel_id = '6199dde695db4ee4ab392222d5af1e5c'
@patch('kolibri.content.management.commands.importchannel.paths.get_content_database_file_url')
@patch('kolibri.content.management.commands.importchannel.paths.get_content_database_file_path')
@patch('kolibri.content.management.commands.importchannel.transfer.FileDownload')
@patch('kolibri.content.management.commands.importchannel.AsyncCommand.cancel', return_value=True)
@patch('kolibri.content.management.commands.importchannel.AsyncCommand.is_cancelled', return_value=True)
def test_remote_cancel_during_transfer(self, is_cancelled_mock, cancel_mock, FileDownloadMock, local_path_mock, remote_path_mock, start_progress_mock,
import_channel_mock):
local_path = tempfile.mkstemp()[1]
local_path_mock.return_value = local_path
remote_path_mock.return_value = 'notest'
FileDownloadMock.return_value.__iter__.return_value = ['one', 'two', 'three']
call_command("importchannel", "network", self.the_channel_id)
# Check that is_cancelled was called
is_cancelled_mock.assert_called_with()
# Check that the FileDownload initiated
FileDownloadMock.assert_called_with('notest', local_path)
# Check that cancel was called
cancel_mock.assert_called_with()
# Test that import channel cleans up database file if cancelled
self.assertFalse(os.path.exists(local_path))
@patch('kolibri.content.management.commands.importchannel.paths.get_content_database_file_path')
@patch('kolibri.content.management.commands.importchannel.transfer.FileCopy')
@patch('kolibri.content.management.commands.importchannel.AsyncCommand.cancel', return_value=True)
@patch('kolibri.content.management.commands.importchannel.AsyncCommand.is_cancelled', return_value=True)
def test_local_cancel_during_transfer(self, is_cancelled_mock, cancel_mock, FileCopyMock, local_path_mock, start_progress_mock, import_channel_mock):
local_dest_path = tempfile.mkstemp()[1]
local_src_path = tempfile.mkstemp()[1]
local_path_mock.side_effect = [local_dest_path, local_src_path]
FileCopyMock.return_value.__iter__.return_value = ['one', 'two', 'three']
call_command("importchannel", "disk", self.the_channel_id, tempfile.mkdtemp())
# Check that is_cancelled was called
is_cancelled_mock.assert_called_with()
# Check that the FileCopy initiated
FileCopyMock.assert_called_with(local_src_path, local_dest_path)
# Check that cancel was called
cancel_mock.assert_called_with()
# Test that import channel cleans up database file if cancelled
self.assertFalse(os.path.exists(local_dest_path))
@patch('kolibri.content.management.commands.importchannel.AsyncCommand.cancel')
@patch('kolibri.content.management.commands.importchannel.AsyncCommand.is_cancelled', return_value=True)
def test_remote_import_sslerror(self, is_cancelled_mock, cancel_mock, start_progress_mock, import_channel_mock):
SSLERROR = SSLError(['SSL routines', 'ssl3_get_record', 'decryption failed or bad record mac'])
if 'OpenSSL' in sys.modules:
from OpenSSL.SSL import Error
SSLERROR = Error(['SSL routines', 'ssl3_get_record', 'decryption failed or bad record mac'])
with patch('kolibri.content.utils.transfer.Transfer.next', side_effect=SSLERROR):
call_command('importchannel', 'network', '197934f144305350b5820c7c4dd8e194')
cancel_mock.assert_called_with()
import_channel_mock.assert_not_called()
@patch('kolibri.content.utils.transfer.Transfer.next', side_effect=ReadTimeout('Read timed out.'))
@patch('kolibri.content.management.commands.importchannel.AsyncCommand.cancel')
@patch('kolibri.content.management.commands.importchannel.AsyncCommand.is_cancelled', return_value=True)
def test_remote_import_readtimeout(self, is_cancelled_mock, cancel_mock, sslerror_mock, start_progress_mock, import_channel_mock):
call_command('importchannel', 'network', '197934f144305350b5820c7c4dd8e194')
cancel_mock.assert_called_with()
import_channel_mock.assert_not_called()
def timed_out(id_token):
raise ReadTimeout('too long')
def send(self, request, **kwargs):
'''Sends a PreparedRequest object over FTP. Returns a response object.
'''
# Get the authentication from the prepared request, if any.
auth = self.get_username_password_from_header(request)
# Next, get the host and the path.
scheme, host, port, path = self.get_host_and_path_from_url(request)
# Sort out the timeout.
timeout = kwargs.get('timeout', None)
# Look for a proxy
proxies = kwargs.get('proxies', {})
proxy = proxies.get(scheme)
# If there is a proxy, then we actually want to make a HTTP request
if proxy:
return self.send_proxy(request, proxy, **kwargs)
# Establish the connection and login if needed.
self.conn = ftplib.FTP()
# Use a flag to distinguish read vs connection timeouts, and a flat set
# of except blocks instead of a nested try-except, because python 3
# exception chaining makes things weird
connected = False
try:
self.conn.connect(host, port, timeout)
connected = True
if auth is not None:
self.conn.login(auth[0], auth[1])
else:
self.conn.login()
# Get the method and attempt to find the function to call.
resp = self.func_table[request.method](path, request)
except socket.timeout as e:
# requests distinguishes between connection timeouts and others
if connected:
raise ReadTimeout(e, request=request)
else:
raise ConnectTimeout(e, request=request)
# ftplib raises EOFError if the connection is unexpectedly closed.
# Convert that or any other socket error to a ConnectionError.
except (EOFError, socket.error) as e:
raise ConnectionError(e, request=request)
# Raised for 5xx errors. FTP uses 550 for both ENOENT and EPERM type
# errors, so just translate all of these into a http-ish 404
except ftplib.error_perm as e:
# The exception message is probably from the server, so if it's
# non-ascii, who knows what the encoding is. Latin1 has the
# advantage of not being able to fail.
resp = build_text_response(request,
BytesIO(str(e).encode('latin1')),
str(codes.not_found))
# 4xx reply, translate to a http 503
except ftplib.error_temp as e:
resp = build_text_response(request,
BytesIO(str(e).encode('latin1')),
str(codes.unavailable))
# error_reply is an unexpected status code, and error_proto is an
# invalid status code. Error is the generic ftplib error, usually
# raised when a line is too long. Translate all of them to a generic
# RequestException
except (ftplib.error_reply, ftplib.error_proto, ftplib.Error) as e:
raise RequestException(e, request=request)
# Return the response.
return resp
def get(self, *args, **kw):
raise ReadTimeout()
def test_raises_on_timeout_error(self):
responses.add(responses.GET, "https://example.com", body=ReadTimeout())
session = Session("https://example.com", max_retries=1)
with self.assertRaises(ReadTimeout):
_ = session.get("")
def timeout_callback(http_request):
if "sleep" in http_request.url:
raise ReadTimeout("Read timed out")
return 200, {}, "Hello there"
class ImportChannelTestCase(TestCase):
"""
Test case for the importchannel management command.
"""
the_channel_id = "6199dde695db4ee4ab392222d5af1e5c"
@patch(
"kolibri.core.content.management.commands.importchannel.paths.get_content_database_file_url"
)
@patch(
"kolibri.core.content.management.commands.importchannel.paths.get_content_database_file_path"
)
@patch(
"kolibri.core.content.management.commands.importchannel.transfer.FileDownload"
)
@patch(
"kolibri.core.content.management.commands.importchannel.AsyncCommand.cancel",
return_value=True,
)
@patch(
"kolibri.core.content.management.commands.importchannel.AsyncCommand.is_cancelled",
return_value=True,
)
def test_remote_cancel_during_transfer(
self,
is_cancelled_mock,
cancel_mock,
FileDownloadMock,
local_path_mock,
remote_path_mock,
start_progress_mock,
import_channel_mock,
):
local_path = tempfile.mkstemp()[1]
local_path_mock.return_value = local_path
remote_path_mock.return_value = "notest"
FileDownloadMock.return_value.__iter__.return_value = ["one", "two", "three"]
call_command("importchannel", "network", self.the_channel_id)
# Check that is_cancelled was called
is_cancelled_mock.assert_called_with()
# Check that the FileDownload initiated
FileDownloadMock.assert_called_with("notest", local_path)
# Check that cancel was called
cancel_mock.assert_called_with()
# Test that import channel cleans up database file if cancelled
self.assertFalse(os.path.exists(local_path))
@patch(
"kolibri.core.content.management.commands.importchannel.paths.get_content_database_file_path"
)
@patch("kolibri.core.content.management.commands.importchannel.transfer.FileCopy")
@patch(
"kolibri.core.content.management.commands.importchannel.AsyncCommand.cancel",
return_value=True,
)
@patch(
"kolibri.core.content.management.commands.importchannel.AsyncCommand.is_cancelled",
return_value=True,
)
def test_local_cancel_during_transfer(
self,
is_cancelled_mock,
cancel_mock,
FileCopyMock,
local_path_mock,
start_progress_mock,
import_channel_mock,
):
local_dest_path = tempfile.mkstemp()[1]
local_src_path = tempfile.mkstemp()[1]
local_path_mock.side_effect = [local_dest_path, local_src_path]
FileCopyMock.return_value.__iter__.return_value = ["one", "two", "three"]
call_command("importchannel", "disk", self.the_channel_id, tempfile.mkdtemp())
# Check that is_cancelled was called
is_cancelled_mock.assert_called_with()
# Check that the FileCopy initiated
FileCopyMock.assert_called_with(local_src_path, local_dest_path)
# Check that cancel was called
cancel_mock.assert_called_with()
# Test that import channel cleans up database file if cancelled
self.assertFalse(os.path.exists(local_dest_path))
@patch("kolibri.core.content.management.commands.importchannel.AsyncCommand.cancel")
@patch(
"kolibri.core.content.management.commands.importchannel.AsyncCommand.is_cancelled",
return_value=True,
)
def test_remote_import_sslerror(
self, is_cancelled_mock, cancel_mock, start_progress_mock, import_channel_mock
):
SSLERROR = SSLError(
["SSL routines", "ssl3_get_record", "decryption failed or bad record mac"]
)
if "OpenSSL" in sys.modules:
from OpenSSL.SSL import Error
SSLERROR = Error(
[
"SSL routines",
"ssl3_get_record",
"decryption failed or bad record mac",
]
)
with patch(
"kolibri.core.content.utils.transfer.Transfer.next", side_effect=SSLERROR
):
call_command("importchannel", "network", "197934f144305350b5820c7c4dd8e194")
cancel_mock.assert_called_with()
import_channel_mock.assert_not_called()
@patch(
"kolibri.core.content.utils.transfer.Transfer.next",
side_effect=ReadTimeout("Read timed out."),
)
@patch("kolibri.core.content.management.commands.importchannel.AsyncCommand.cancel")
@patch(
"kolibri.core.content.management.commands.importchannel.AsyncCommand.is_cancelled",
return_value=True,
)
def test_remote_import_readtimeout(
self,
is_cancelled_mock,
cancel_mock,
sslerror_mock,
start_progress_mock,
import_channel_mock,
):
call_command("importchannel", "network", "197934f144305350b5820c7c4dd8e194")
cancel_mock.assert_called_with()
import_channel_mock.assert_not_called()
def send(self,
request,
stream=False,
timeout=None,
verify=True,
cert=None,
proxies=None):
"""Follows `requests.HTTPAdapter.send()`, but does not send the last
couple of bytes, instead storing them in `self._pending_requests`.
"""
# TODO: ensure no pooling happens here
try:
conn = self.get_connection(request.url, proxies)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies)
if isinstance(timeout, tuple):
try:
connect, read = timeout
timeout = TimeoutSauce(connect=connect, read=read)
except ValueError as e:
# this may raise a string formatting error.
err = ("Invalid timeout {}. Pass a (connect, read) "
"timeout tuple, or a single float to set "
"both timeouts to the same value".format(timeout))
raise ValueError(err)
elif isinstance(timeout, TimeoutSauce):
pass
else:
timeout = TimeoutSauce(connect=timeout, read=timeout)
# this is what we will return for now. this object will later be
# overwritten after the request is finished.
response = Response()
self.build_dummy_response_into(response, request)
try:
# Send the request.
if hasattr(conn, 'proxy_pool'):
conn = conn.proxy_pool
low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT)
try:
low_conn.putrequest(request.method,
url,
skip_accept_encoding=True)
for header, value in request.headers.items():
low_conn.putheader(header, value)
if request.body is None:
# no body
# MASSIVE HACK ALERT:
# at this point, *no* information has been sent to the
# server---it is buffered and will only be sent after we
# call low_conn.endheaders(). however, this request has no
# body, so calling low_conn.endheaders() would actually
# cause the request to be processed by the server
# immediately. endheaders() is the only public way to get
# the connection into a 'sent' state, which is the only
# state in which you are allowed to read the response. so
# we dig into the internals of the HTTPConnection object to
# get it into the 'sent' state and send *most* of its
# internal buffer without sending the two final newlines.
# this follows HTTPConnection._send_output
msg = b"\r\n".join(low_conn._buffer)
del low_conn._buffer[:]
low_conn.send(msg)
low_conn._HTTPConnection__state = 'Request-sent'
self._pending_requests.append(
(request, low_conn, b'\r\n\r\n', response))
else:
# some body, can end headers now
low_conn.endheaders()
body = request.body
if isinstance(body, str):
body = body.encode('utf-8')
if 'Content-Length' in request.headers:
# single body
low_conn.send(body[:-3])
self._pending_requests.append(
(request, low_conn, body[-3:], response))
else:
# chunked body
for i in request.body:
low_conn.send(hex(len(i))[2:].encode('utf-8'))
low_conn.send(b'\r\n')
low_conn.send(i)
low_conn.send(b'\r\n')
self._pending_requests.append(
(request, low_conn, b'0\r\n\r\n', response))
except:
# If we hit any problems here, clean up the connection.
# Then, reraise so that we can handle the actual exception.
low_conn.close()
raise
except (ProtocolError, socket.error) as err:
raise ConnectionError(err, request=request)
except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
# TODO: Remove this in 3.0.0: see #2811
if not isinstance(e.reason, NewConnectionError):
raise ConnectTimeout(e, request=request)
if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)
if isinstance(e.reason, _ProxyError):
raise ProxyError(e, request=request)
if isinstance(e.reason, _SSLError):
# This branch is for urllib3 v1.22 and later.
raise SSLError(e, request=request)
raise ConnectionError(e, request=request)
except ClosedPoolError as e:
raise ConnectionError(e, request=request)
except _ProxyError as e:
raise ProxyError(e)
except (_SSLError, _HTTPError) as e:
if isinstance(e, _SSLError):
# This branch is for urllib3 versions earlier than v1.22
raise SSLError(e, request=request)
elif isinstance(e, ReadTimeoutError):
raise ReadTimeout(e, request=request)
else:
raise
return response
def mocked_get(url, **kwargs):
assert '?access_token=' in url
raise ReadTimeout('too long')
def request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=None,
cert=None,
json=None,
**kwargs):
"""Constructs a :class:`Request <Request>`, prepares it and sends it.
Returns :class:`Response <Response>` object.
:param method: method for the new :class:`Request` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary or bytes to be sent in the query
string for the :class:`Request`.
:param data: (optional) Dictionary, bytes, or file-like object to send
in the body of the :class:`Request`.
:param json: (optional) json to send in the body of the
:class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the
:class:`Request`.
:param cookies: (optional) Dict or CookieJar object to send with the
:class:`Request`.
:param files: (optional) Dictionary of ``'filename': file-like-objects``
for multipart encoding upload.
:param auth: (optional) Auth tuple or callable to enable
Basic/Digest/Custom HTTP Auth.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple
:param allow_redirects: (optional) Set to True by default.
:type allow_redirects: bool
:param proxies: (optional) Dictionary mapping protocol or protocol and
hostname to the URL of the proxy.
:param stream: (optional) whether to immediately download the response
content. Defaults to ``False``.
:param verify: (optional) whether the SSL cert will be verified.
A CA_BUNDLE path can also be provided. Defaults to ``True``.
:param cert: (optional) if String, path to ssl client cert file (.pem).
If Tuple, ('cert', 'key') pair.
:rtype: requests.Response
"""
#===============================================================================================================
# add by mz
error_type = kwargs.get("error_type")
if error_type:
from requests.exceptions import InvalidURL, URLRequired, ConnectTimeout, ConnectionError, SSLError, ReadTimeout
from requests.exceptions import InvalidSchema, MissingSchema, ChunkedEncodingError, ContentDecodingError
from requests.exceptions import RequestException, HTTPError, ProxyError, Timeout, RetryError, StreamConsumedError
from requests.exceptions import TooManyRedirects
get_error = {
"InvalidURL": InvalidURL(),
"URLRequired": URLRequired(),
"ConnectTimeout": ConnectTimeout(),
"ConnectionError": ConnectionError(),
"SSLError": SSLError(),
"ReadTimeout": ReadTimeout(),
"InvalidSchema": InvalidSchema(),
"MissingSchema": MissingSchema(),
"ChunkedEncodingError": ChunkedEncodingError(),
"ContentDecodingError": ContentDecodingError(),
"StreamConsumedError": StreamConsumedError(),
"TooManyRedirects": TooManyRedirects(),
"RequestException": RequestException(),
"HTTPError": HTTPError(),
"ProxyError": ProxyError(),
"Timeout": Timeout(),
"RetryError": RetryError
}
error_ = get_error[error_type]
raise error_
#===============================================================================================================
# Create the Request
req = Request(
method=method.upper(),
url=url,
headers=headers,
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
proxies = proxies or {}
settings = self.merge_environment_settings(prep.url, proxies, stream,
verify, cert)
# Send the request.
send_kwargs = {
'timeout': timeout,
'allow_redirects': allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
return resp