def test_set_proxy(self, _):
manager = connections.ConnectionsManager(proxy="http://localhost")
self._check_proxies(manager, "http://localhost", "http://localhost")
manager = connections.ConnectionsManager(
proxy="http://localhost", secure_proxy="https://localhost")
self._check_proxies(manager, "http://localhost", "https://localhost")
manager = connections.ConnectionsManager(retries_num=2)
self.assertEqual(2, manager.retries_num)
for h in manager.opener.handlers:
if isinstance(h, connections.RetryHandler):
break
else:
self.fail("RetryHandler should be in list of handlers.")
def test_open_stream(self, *_):
manager = connections.ConnectionsManager(retries_num=2)
manager.open_stream("/test/file")
self.assertEqual(1, manager.opener.open.call_count)
args = manager.opener.open.call_args[0]
self.assertIsInstance(args[0], connections.RetryableRequest)
self.assertEqual(2, args[0].retries_left)
def test_raise_other_errors(self, *_):
manager = connections.ConnectionsManager()
manager.opener.open.side_effect = \
connections.urlerror.HTTPError("", 500, "", {}, None)
with self.assertRaises(connections.urlerror.URLError):
manager.open_stream("/test/file")
self.assertEqual(1, manager.opener.open.call_count)
def test_make_request(self, *_):
manager = connections.ConnectionsManager(retries_num=2)
request = manager.make_request("/test/file", 0)
self.assertIsInstance(request, connections.RetryableRequest)
self.assertEqual("file:///test/file", request.get_full_url())
self.assertEqual(0, request.offset)
self.assertEqual(2, request.retries_left)
request2 = manager.make_request("http://server/path", 100)
self.assertEqual("http://server/path", request2.get_full_url())
self.assertEqual(100, request2.offset)
def test_progressive_delay_between_request(self, time_mock, *_):
manager = connections.ConnectionsManager(retries_num=6,
retry_interval=1)
manager.opener.open.side_effect = IOError("I/O Error")
with self.assertRaises(IOError):
manager.open_stream("/test/file")
self.assertEqual(7, manager.opener.open.call_count)
self.assertEqual([1, 1, 2, 3, 4, 5],
[x[0][0] for x in time_mock.sleep.call_args_list])
def test_retrieve_non_existence(self, os, *_):
manager = connections.ConnectionsManager()
os.stat.side_effect = OSError(2, "")
os.open.return_value = 1
response = mock.MagicMock()
manager.opener.open.return_value = response
response.read.side_effect = [b"test", b""]
manager.retrieve("/file/src", "/file/dst", size=20)
os.lseek.assert_called_once_with(1, 0, os.SEEK_SET)
os.ftruncate.assert_called_once_with(1, 0)
self.assertEqual(1, os.write.call_count)
os.fsync.assert_called_once_with(1)
os.close.assert_called_once_with(1)
def test_retries_on_50x(self, logger, http_open, http_request):
request = connections.RetryableRequest("http:///localhost/file1.txt")
request.retries_left = 1
http_request.return_value = request
response_mock = mock.MagicMock(code=501, msg="not found")
response_mock.getcode.return_value = response_mock.code
http_open.return_value = response_mock
manager = connections.ConnectionsManager(retries_num=2)
with self.assertRaises(connections.urlerror.HTTPError) as trapper:
manager.open_stream("http:///localhost/file1.txt")
self.assertEqual(501, trapper.exception.code)
self.assertEqual(2, http_request.call_count)
for retry_num in six.moves.range(1):
logger.error.assert_any_call(
"request failed: %s - %d(%s), retries left - %d.", mock.ANY,
501, mock.ANY, retry_num)
def test_retries_on_io_error(self, _, logger):
manager = connections.ConnectionsManager(retries_num=2)
manager.opener.open.side_effect = [
IOError("I/O error"), mock.MagicMock()
]
manager.open_stream("/test/file")
self.assertEqual(2, manager.opener.open.call_count)
logger.exception.assert_called_with(
"Failed to open url - %s: %s. retries left - %d.", "/test/file",
"I/O error", 1)
manager.opener.open.side_effect = IOError("I/O error")
with self.assertRaises(IOError):
manager.open_stream("/test/file")
logger.exception.assert_called_with(
"Failed to open url - %s: %s. retries left - %d.", "/test/file",
"I/O error", 0)
def test_retrieve_from_offset_fail(self, os, logger):
manager = connections.ConnectionsManager(retries_num=2)
os.stat.return_value = mock.MagicMock(st_size=10)
os.open.return_value = 1
response = mock.MagicMock()
manager.opener.open.side_effect = [
connections.RangeError("error"), response
]
response.read.side_effect = [b"test", b""]
manager.retrieve("/file/src", "/file/dst", size=20)
logger.warning.assert_called_once_with(
"Failed to resume download, starts from the beginning: %s",
"/file/src")
os.lseek.assert_called_once_with(1, 0, os.SEEK_SET)
os.ftruncate.assert_called_once_with(1, 0)
self.assertEqual(1, os.write.call_count)
os.fsync.assert_called_once_with(1)
os.close.assert_called_once_with(1)