def test_set_proxy(self, _): manager = connections.ConnectionsManager(proxy="http://localhost") self._check_proxies(manager, "http://localhost", "http://localhost") manager = connections.ConnectionsManager( proxy="http://localhost", secure_proxy="https://localhost") self._check_proxies(manager, "http://localhost", "https://localhost") manager = connections.ConnectionsManager(retries_num=2) self.assertEqual(2, manager.retries_num) for h in manager.opener.handlers: if isinstance(h, connections.RetryHandler): break else: self.fail("RetryHandler should be in list of handlers.")
def test_open_stream(self, *_): manager = connections.ConnectionsManager(retries_num=2) manager.open_stream("/test/file") self.assertEqual(1, manager.opener.open.call_count) args = manager.opener.open.call_args[0] self.assertIsInstance(args[0], connections.RetryableRequest) self.assertEqual(2, args[0].retries_left)
def test_raise_other_errors(self, *_): manager = connections.ConnectionsManager() manager.opener.open.side_effect = \ connections.urlerror.HTTPError("", 500, "", {}, None) with self.assertRaises(connections.urlerror.URLError): manager.open_stream("/test/file") self.assertEqual(1, manager.opener.open.call_count)
def test_make_request(self, *_): manager = connections.ConnectionsManager(retries_num=2) request = manager.make_request("/test/file", 0) self.assertIsInstance(request, connections.RetryableRequest) self.assertEqual("file:///test/file", request.get_full_url()) self.assertEqual(0, request.offset) self.assertEqual(2, request.retries_left) request2 = manager.make_request("http://server/path", 100) self.assertEqual("http://server/path", request2.get_full_url()) self.assertEqual(100, request2.offset)
def test_progressive_delay_between_request(self, time_mock, *_): manager = connections.ConnectionsManager(retries_num=6, retry_interval=1) manager.opener.open.side_effect = IOError("I/O Error") with self.assertRaises(IOError): manager.open_stream("/test/file") self.assertEqual(7, manager.opener.open.call_count) self.assertEqual([1, 1, 2, 3, 4, 5], [x[0][0] for x in time_mock.sleep.call_args_list])
def test_retrieve_non_existence(self, os, *_): manager = connections.ConnectionsManager() os.stat.side_effect = OSError(2, "") os.open.return_value = 1 response = mock.MagicMock() manager.opener.open.return_value = response response.read.side_effect = [b"test", b""] manager.retrieve("/file/src", "/file/dst", size=20) os.lseek.assert_called_once_with(1, 0, os.SEEK_SET) os.ftruncate.assert_called_once_with(1, 0) self.assertEqual(1, os.write.call_count) os.fsync.assert_called_once_with(1) os.close.assert_called_once_with(1)
def test_retries_on_50x(self, logger, http_open, http_request): request = connections.RetryableRequest("http:///localhost/file1.txt") request.retries_left = 1 http_request.return_value = request response_mock = mock.MagicMock(code=501, msg="not found") response_mock.getcode.return_value = response_mock.code http_open.return_value = response_mock manager = connections.ConnectionsManager(retries_num=2) with self.assertRaises(connections.urlerror.HTTPError) as trapper: manager.open_stream("http:///localhost/file1.txt") self.assertEqual(501, trapper.exception.code) self.assertEqual(2, http_request.call_count) for retry_num in six.moves.range(1): logger.error.assert_any_call( "request failed: %s - %d(%s), retries left - %d.", mock.ANY, 501, mock.ANY, retry_num)
def test_retries_on_io_error(self, _, logger): manager = connections.ConnectionsManager(retries_num=2) manager.opener.open.side_effect = [ IOError("I/O error"), mock.MagicMock() ] manager.open_stream("/test/file") self.assertEqual(2, manager.opener.open.call_count) logger.exception.assert_called_with( "Failed to open url - %s: %s. retries left - %d.", "/test/file", "I/O error", 1) manager.opener.open.side_effect = IOError("I/O error") with self.assertRaises(IOError): manager.open_stream("/test/file") logger.exception.assert_called_with( "Failed to open url - %s: %s. retries left - %d.", "/test/file", "I/O error", 0)
def test_retrieve_from_offset_fail(self, os, logger): manager = connections.ConnectionsManager(retries_num=2) os.stat.return_value = mock.MagicMock(st_size=10) os.open.return_value = 1 response = mock.MagicMock() manager.opener.open.side_effect = [ connections.RangeError("error"), response ] response.read.side_effect = [b"test", b""] manager.retrieve("/file/src", "/file/dst", size=20) logger.warning.assert_called_once_with( "Failed to resume download, starts from the beginning: %s", "/file/src") os.lseek.assert_called_once_with(1, 0, os.SEEK_SET) os.ftruncate.assert_called_once_with(1, 0) self.assertEqual(1, os.write.call_count) os.fsync.assert_called_once_with(1) os.close.assert_called_once_with(1)