Exemplo n.º 1
0
    def setUp(self):
        self.send_called = 0
        StorageDriver.connectionCls.conn_classes = (None, StorageMockHttp)

        self.driver1 = StorageDriver('username', 'key', host='localhost')
        self.driver1.supports_chunked_encoding = True
        self.driver2 = StorageDriver('username', 'key', host='localhost')
        self.driver2.supports_chunked_encoding = False
Exemplo n.º 2
0
    def setUp(self):
        self.send_called = 0
        StorageDriver.connectionCls.conn_class = BaseMockRawResponse

        self.driver1 = StorageDriver("username", "key", host="localhost")
        self.driver1.supports_chunked_encoding = True

        self.driver2 = StorageDriver("username", "key", host="localhost")
        self.driver2.supports_chunked_encoding = False

        self.driver1.strict_mode = False
        self.driver1.strict_mode = False
Exemplo n.º 3
0
    def setUp(self):
        self.send_called = 0
        StorageDriver.connectionCls.conn_class = StorageMockHttp
        StorageDriver.connectionCls.rawResponseCls = BaseMockRawResponse

        self.driver1 = StorageDriver('username', 'key', host='localhost')
        self.driver1.supports_chunked_encoding = True

        self.driver2 = StorageDriver('username', 'key', host='localhost')
        self.driver2.supports_chunked_encoding = False

        self.driver1.strict_mode = False
        self.driver1.strict_mode = False
    def setUp(self):
        self.send_called = 0
        StorageDriver.connectionCls.conn_classes = (None, StorageMockHttp)

        self.driver1 = StorageDriver('username', 'key', host='localhost')
        self.driver1.supports_chunked_encoding = True
        self.driver2 = StorageDriver('username', 'key', host='localhost')
        self.driver2.supports_chunked_encoding = False
Exemplo n.º 5
0
    def setUp(self):
        self.send_called = 0
        StorageDriver.connectionCls.conn_class = StorageMockHttp
        StorageDriver.connectionCls.rawResponseCls = BaseMockRawResponse

        self.driver1 = StorageDriver('username', 'key', host='localhost')
        self.driver1.supports_chunked_encoding = True

        self.driver2 = StorageDriver('username', 'key', host='localhost')
        self.driver2.supports_chunked_encoding = False

        self.driver1.strict_mode = False
        self.driver1.strict_mode = False
Exemplo n.º 6
0
 def __init__(self, key, secret=None, secure=True, host=None, port=None):
     self._credentials = SharedKeyCredentials(key, secret)
     StorageDriver.__init__(self, key, secret, secure, host, port)
Exemplo n.º 7
0
class BaseStorageTests(unittest.TestCase):
    def setUp(self):
        self.send_called = 0
        StorageDriver.connectionCls.conn_classes = (None, StorageMockHttp)

        self.driver1 = StorageDriver('username', 'key', host='localhost')
        self.driver1.supports_chunked_encoding = True
        self.driver2 = StorageDriver('username', 'key', host='localhost')
        self.driver2.supports_chunked_encoding = False

        self.driver1.strict_mode = False
        self.driver1.strict_mode = False

    def test__upload_object_iterator_must_have_next_method(self):
        class Iterator(object):
            def next(self):
                pass

        class Iterator2(file):
            def __init__(self):
                pass

        class SomeClass(object):
            pass

        valid_iterators = [Iterator(), Iterator2(), StringIO('bar')]
        invalid_iterators = ['foobar', '', False, True, 1, object()]

        def upload_func(*args, **kwargs):
            return True, 'barfoo', 100

        kwargs = {
            'object_name': 'foo',
            'content_type': 'foo/bar',
            'upload_func': upload_func,
            'upload_func_kwargs': {},
            'request_path': '/',
            'headers': {}
        }

        for value in valid_iterators:
            kwargs['iterator'] = value
            self.driver1._upload_object(**kwargs)

        for value in invalid_iterators:
            kwargs['iterator'] = value

            try:
                self.driver1._upload_object(**kwargs)
            except AttributeError:
                pass
            else:
                self.fail('Exception was not thrown')

    def test_upload_zero_bytes_long_object_via_stream(self):
        iterator = Mock()

        if PY3:
            iterator.__next__ = Mock()
            iterator.__next__.side_effect = StopIteration()
        else:
            iterator.next.side_effect = StopIteration()

        def mock_send(data):
            self.send_called += 1

        response = Mock()
        response.connection.connection.send = mock_send

        # Normal
        success, data_hash, bytes_transferred = \
            self.driver1._stream_data(response=response,
                                      iterator=iterator,
                                      chunked=False, calculate_hash=True)

        self.assertTrue(success)
        self.assertEqual(data_hash, hashlib.md5(b('')).hexdigest())
        self.assertEqual(bytes_transferred, 0)
        self.assertEqual(self.send_called, 1)

        # Chunked
        success, data_hash, bytes_transferred = \
            self.driver1._stream_data(response=response,
                                      iterator=iterator,
                                      chunked=True, calculate_hash=True)

        self.assertTrue(success)
        self.assertEqual(data_hash, hashlib.md5(b('')).hexdigest())
        self.assertEqual(bytes_transferred, 0)
        self.assertEqual(self.send_called, 5)

    def test__upload_data(self):
        def mock_send(data):
            self.send_called += 1

        response = Mock()
        response.connection.connection.send = mock_send

        data = '123456789901234567'
        success, data_hash, bytes_transferred = \
            self.driver1._upload_data(response=response, data=data,
                                      calculate_hash=True)

        self.assertTrue(success)
        self.assertEqual(data_hash, hashlib.md5(b(data)).hexdigest())
        self.assertEqual(bytes_transferred, (len(data)))
        self.assertEqual(self.send_called, 1)

    def test__get_hash_function(self):
        self.driver1.hash_type = 'md5'
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        self.driver1.hash_type = 'sha1'
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        try:
            self.driver1.hash_type = 'invalid-hash-function'
            func = self.driver1._get_hash_function()
        except RuntimeError:
            pass
        else:
            self.fail('Invalid hash type but exception was not thrown')

    def test_upload_no_content_type_supplied_or_detected(self):
        iterator = StringIO()

        upload_func = Mock()
        upload_func.return_value = True, '', 0

        # strict_mode is disabled, default content type should be used
        self.driver1.connection = Mock()

        self.driver1._upload_object(object_name='test',
                                    content_type=None,
                                    upload_func=upload_func,
                                    upload_func_kwargs={},
                                    request_path='/',
                                    iterator=iterator)

        headers = self.driver1.connection.request.call_args[-1]['headers']
        self.assertEqual(headers['Content-Type'], DEFAULT_CONTENT_TYPE)

        # strict_mode is enabled, exception should be thrown

        self.driver1.strict_mode = True
        expected_msg = ('File content-type could not be guessed and no'
                        ' content_type value is provided')
        self.assertRaisesRegexp(AttributeError,
                                expected_msg,
                                self.driver1._upload_object,
                                object_name='test',
                                content_type=None,
                                upload_func=upload_func,
                                upload_func_kwargs={},
                                request_path='/',
                                iterator=iterator)
Exemplo n.º 8
0
 def setUp(self):
     self.send_called = 0
     StorageDriver.connectionCls.conn_classes = (None, StorageMockHttp)
     self.driver = StorageDriver('username', 'key', host='localhost')
Exemplo n.º 9
0
class BaseStorageTests(unittest.TestCase):
    def setUp(self):
        self.send_called = 0
        StorageDriver.connectionCls.conn_classes = (None, StorageMockHttp)
        self.driver = StorageDriver('username', 'key', host='localhost')

    def test__upload_object_iterator_must_have_next_method(self):
        class Iterator(object):
            def next(self):
                pass

        class Iterator2(file):
            def __init__(self):
                pass

        class SomeClass(object):
            pass

        valid_iterators = [ Iterator(), Iterator2(), StringIO('bar') ]
        invalid_iterators = [ 'foobar', '', False, True, 1, object() ]

        def upload_func(*args, **kwargs):
            return True, 'barfoo', 100

        kwargs = {'object_name': 'foo', 'content_type': 'foo/bar',
                   'upload_func': upload_func, 'upload_func_kwargs': {},
                   'request_path': '/', 'headers': {}}

        for value in valid_iterators:
            kwargs['iterator'] = value
            self.driver._upload_object(**kwargs)

        for value in invalid_iterators:
            kwargs['iterator'] = value

            try:
                self.driver._upload_object(**kwargs)
            except AttributeError:
                pass
            else:
                self.fail('Exception was not thrown')

    def test_upload_zero_bytes_long_object_via_stream(self):
        iterator = Mock()
        iterator.next.side_effect = StopIteration()

        def mock_send(data):
            self.send_called += 1

        response = Mock()
        response.connection.connection.send = mock_send

        # Normal
        success, data_hash, bytes_transferred = \
                 self.driver._stream_data(response=response, iterator=iterator,
                                          chunked=False, calculate_hash=True)

        self.assertTrue(success)
        self.assertEqual(data_hash, hashlib.md5('').hexdigest())
        self.assertEqual(bytes_transferred, 0)
        self.assertEqual(self.send_called, 1)

        # Chunked
        success, data_hash, bytes_transferred = \
                 self.driver._stream_data(response=response, iterator=iterator,
                                          chunked=True, calculate_hash=True)

        self.assertTrue(success)
        self.assertEqual(data_hash, hashlib.md5('').hexdigest())
        self.assertEqual(bytes_transferred, 0)
        self.assertEqual(self.send_called, 5)
Exemplo n.º 10
0
class BaseStorageTests(unittest.TestCase):
    def setUp(self):
        self.send_called = 0
        StorageDriver.connectionCls.conn_class = BaseMockRawResponse

        self.driver1 = StorageDriver("username", "key", host="localhost")
        self.driver1.supports_chunked_encoding = True

        self.driver2 = StorageDriver("username", "key", host="localhost")
        self.driver2.supports_chunked_encoding = False

        self.driver1.strict_mode = False
        self.driver1.strict_mode = False

    def test__upload_object_iterator_must_have_next_method(self):

        valid_iterators = [BytesIO(b("134")), StringIO("bar")]
        invalid_iterators = ["foobar", "", False, True, 1, object()]

        kwargs = {
            "object_name": "foo",
            "content_type": "foo/bar",
            "request_path": "/",
            "headers": {},
        }

        for value in valid_iterators:
            kwargs["stream"] = value
            self.driver1._upload_object(**kwargs)

        for value in invalid_iterators:
            kwargs["stream"] = value

            try:
                self.driver1._upload_object(**kwargs)
            except AttributeError:
                pass
            else:
                self.fail("Exception was not thrown")

    def test__upload_object_does_not_stream_response(self):
        resp = self.driver1._upload_object(
            object_name="foo",
            content_type="foo/bar",
            request_path="/",
            stream=iter(b"foo"),
        )
        mock_response = resp["response"].response._response
        response_streamed = mock_response.request.stream
        assert response_streamed is False

    def test__get_hash_function(self):
        self.driver1.hash_type = "md5"
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        self.driver1.hash_type = "sha1"
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        try:
            self.driver1.hash_type = "invalid-hash-function"
            func = self.driver1._get_hash_function()
        except RuntimeError:
            pass
        else:
            self.fail("Invalid hash type but exception was not thrown")

    def test_upload_no_content_type_supplied_or_detected(self):
        iterator = StringIO()

        # strict_mode is disabled, default content type should be used
        self.driver1.connection = Mock()

        self.driver1._upload_object(
            object_name="test", content_type=None, request_path="/", stream=iterator
        )

        headers = self.driver1.connection.request.call_args[-1]["headers"]
        self.assertEqual(headers["Content-Type"], DEFAULT_CONTENT_TYPE)

        # strict_mode is enabled, exception should be thrown

        self.driver1.strict_mode = True
        expected_msg = (
            'File content-type could not be guessed for "test" '
            "and no content_type value is provided"
        )
        assertRaisesRegex(
            self,
            AttributeError,
            expected_msg,
            self.driver1._upload_object,
            object_name="test",
            content_type=None,
            request_path="/",
            stream=iterator,
        )

    @mock.patch("libcloud.utils.files.exhaust_iterator")
    @mock.patch("libcloud.utils.files.read_in_chunks")
    def test_upload_object_hash_calculation_is_efficient(
        self, mock_read_in_chunks, mock_exhaust_iterator
    ):
        # Verify that we don't buffer whole file in memory when calculating
        # object has when iterator has __next__ method, but instead read and
        # calculate hash in chunks
        size = 100

        self.driver1.connection = Mock()

        # stream has __next__ method and next() method
        mock_read_in_chunks.return_value = "a" * size

        iterator = BodyStream("a" * size)
        self.assertTrue(hasattr(iterator, "__next__"))
        self.assertTrue(hasattr(iterator, "next"))

        self.assertEqual(mock_read_in_chunks.call_count, 0)
        self.assertEqual(mock_exhaust_iterator.call_count, 0)

        result = self.driver1._upload_object(
            object_name="test1", content_type=None, request_path="/", stream=iterator
        )

        hasher = hashlib.md5()
        hasher.update(b("a") * size)
        expected_hash = hasher.hexdigest()

        self.assertEqual(result["data_hash"], expected_hash)
        self.assertEqual(result["bytes_transferred"], size)

        headers = self.driver1.connection.request.call_args[-1]["headers"]
        self.assertEqual(headers["Content-Type"], DEFAULT_CONTENT_TYPE)

        self.assertEqual(mock_read_in_chunks.call_count, 1)
        self.assertEqual(mock_exhaust_iterator.call_count, 0)

        # stream has only has next() method
        mock_read_in_chunks.return_value = "b" * size

        iterator = iter([str(v) for v in ["b" * size]])

        if PY2:
            self.assertFalse(hasattr(iterator, "__next__"))
            self.assertTrue(hasattr(iterator, "next"))
        else:
            self.assertTrue(hasattr(iterator, "__next__"))
            self.assertFalse(hasattr(iterator, "next"))

        self.assertEqual(mock_read_in_chunks.call_count, 1)
        self.assertEqual(mock_exhaust_iterator.call_count, 0)

        self.assertEqual(mock_read_in_chunks.call_count, 1)
        self.assertEqual(mock_exhaust_iterator.call_count, 0)

        result = self.driver1._upload_object(
            object_name="test2", content_type=None, request_path="/", stream=iterator
        )

        hasher = hashlib.md5()
        hasher.update(b("b") * size)
        expected_hash = hasher.hexdigest()

        self.assertEqual(result["data_hash"], expected_hash)
        self.assertEqual(result["bytes_transferred"], size)

        headers = self.driver1.connection.request.call_args[-1]["headers"]
        self.assertEqual(headers["Content-Type"], DEFAULT_CONTENT_TYPE)

        self.assertEqual(mock_read_in_chunks.call_count, 2)
        self.assertEqual(mock_exhaust_iterator.call_count, 0)

    def test_upload_object_via_stream_illegal_seek_errors_are_ignored(self):
        # Illegal seek errors should be ignored
        size = 100

        self.driver1.connection = Mock()

        seek_error = OSError("Illegal seek")
        seek_error.errno = 29
        assert errno.ESPIPE == 29

        iterator = BodyStream("a" * size)
        iterator.seek = mock.Mock(side_effect=seek_error)

        result = self.driver1._upload_object(
            object_name="test1", content_type=None, request_path="/", stream=iterator
        )

        hasher = hashlib.md5()
        hasher.update(b("a") * size)
        expected_hash = hasher.hexdigest()

        self.assertEqual(result["data_hash"], expected_hash)
        self.assertEqual(result["bytes_transferred"], size)

        # But others shouldn't
        self.driver1.connection = Mock()

        seek_error = OSError("Other error")
        seek_error.errno = 21

        iterator = BodyStream("b" * size)
        iterator.seek = mock.Mock(side_effect=seek_error)

        self.assertRaisesRegex(
            OSError,
            "Other error",
            self.driver1._upload_object,
            object_name="test1",
            content_type=None,
            request_path="/",
            stream=iterator,
        )

    def test_get_standard_range_str(self):
        result = self.driver1._get_standard_range_str(0, 5)
        self.assertEqual(result, "bytes=0-4")

        result = self.driver1._get_standard_range_str(0)
        self.assertEqual(result, "bytes=0-")
        result = self.driver1._get_standard_range_str(0, 1)

        self.assertEqual(result, "bytes=0-0")

        result = self.driver1._get_standard_range_str(200)
        self.assertEqual(result, "bytes=200-")

        result = self.driver1._get_standard_range_str(10, 200)
        self.assertEqual(result, "bytes=10-199")

        result = self.driver1._get_standard_range_str(10, 11)
        self.assertEqual(result, "bytes=10-10")

        result = self.driver1._get_standard_range_str(10, 11, True)
        self.assertEqual(result, "bytes=10-11")

    @mock.patch("os.environ", {"LIBCLOUD_RETRY_FAILED_HTTP_REQUESTS": True})
    def test_should_retry_rate_limited_errors(self):
        class SecondException(Exception):
            pass

        count = 0

        def raise_on_second(*_, **__):
            nonlocal count
            count += 1
            if count > 1:
                raise SecondException()
            else:
                raise RateLimitReachedError()

        send_mock = Mock()
        self.driver1.connection.connection.session.send = send_mock
        send_mock.side_effect = raise_on_second
        with self.assertRaises(SecondException):
            self.driver1._upload_object(
                object_name="some name",
                content_type="something",
                request_path="/",
                stream=iter([]),
            )
Exemplo n.º 11
0
class BaseStorageTests(unittest.TestCase):

    def setUp(self):
        self.send_called = 0
        StorageDriver.connectionCls.conn_classes = (None, StorageMockHttp)

        self.driver1 = StorageDriver('username', 'key', host='localhost')
        self.driver1.supports_chunked_encoding = True
        self.driver2 = StorageDriver('username', 'key', host='localhost')
        self.driver2.supports_chunked_encoding = False

        self.driver1.strict_mode = False
        self.driver1.strict_mode = False

    def test__upload_object_iterator_must_have_next_method(self):
        class Iterator(object):

            def next(self):
                pass

        class Iterator2(file):

            def __init__(self):
                pass

        class SomeClass(object):
            pass

        valid_iterators = [Iterator(), Iterator2(), StringIO('bar')]
        invalid_iterators = ['foobar', '', False, True, 1, object()]

        def upload_func(*args, **kwargs):
            return True, 'barfoo', 100

        kwargs = {'object_name': 'foo', 'content_type': 'foo/bar',
                  'upload_func': upload_func, 'upload_func_kwargs': {},
                  'request_path': '/', 'headers': {}}

        for value in valid_iterators:
            kwargs['iterator'] = value
            self.driver1._upload_object(**kwargs)

        for value in invalid_iterators:
            kwargs['iterator'] = value

            try:
                self.driver1._upload_object(**kwargs)
            except AttributeError:
                pass
            else:
                self.fail('Exception was not thrown')

    def test_upload_zero_bytes_long_object_via_stream(self):
        iterator = Mock()

        if PY3:
            iterator.__next__ = Mock()
            iterator.__next__.side_effect = StopIteration()
        else:
            iterator.next.side_effect = StopIteration()

        def mock_send(data):
            self.send_called += 1

        response = Mock()
        response.connection.connection.send = mock_send

        # Normal
        success, data_hash, bytes_transferred = \
            self.driver1._stream_data(response=response,
                                      iterator=iterator,
                                      chunked=False, calculate_hash=True)

        self.assertTrue(success)
        self.assertEqual(data_hash, hashlib.md5(b('')).hexdigest())
        self.assertEqual(bytes_transferred, 0)
        self.assertEqual(self.send_called, 1)

        # Chunked
        success, data_hash, bytes_transferred = \
            self.driver1._stream_data(response=response,
                                      iterator=iterator,
                                      chunked=True, calculate_hash=True)

        self.assertTrue(success)
        self.assertEqual(data_hash, hashlib.md5(b('')).hexdigest())
        self.assertEqual(bytes_transferred, 0)
        self.assertEqual(self.send_called, 5)

    def test__upload_data(self):
        def mock_send(data):
            self.send_called += 1

        response = Mock()
        response.connection.connection.send = mock_send

        data = '123456789901234567'
        success, data_hash, bytes_transferred = \
            self.driver1._upload_data(response=response, data=data,
                                      calculate_hash=True)

        self.assertTrue(success)
        self.assertEqual(data_hash, hashlib.md5(b(data)).hexdigest())
        self.assertEqual(bytes_transferred, (len(data)))
        self.assertEqual(self.send_called, 1)

    def test__get_hash_function(self):
        self.driver1.hash_type = 'md5'
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        self.driver1.hash_type = 'sha1'
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        try:
            self.driver1.hash_type = 'invalid-hash-function'
            func = self.driver1._get_hash_function()
        except RuntimeError:
            pass
        else:
            self.fail('Invalid hash type but exception was not thrown')

    def test_upload_no_content_type_supplied_or_detected(self):
        iterator = StringIO()

        upload_func = Mock()
        upload_func.return_value = True, '', 0

        # strict_mode is disabled, default content type should be used
        self.driver1.connection = Mock()

        self.driver1._upload_object(object_name='test',
                                    content_type=None,
                                    upload_func=upload_func,
                                    upload_func_kwargs={},
                                    request_path='/',
                                    iterator=iterator)

        headers = self.driver1.connection.request.call_args[-1]['headers']
        self.assertEqual(headers['Content-Type'], DEFAULT_CONTENT_TYPE)

        # strict_mode is enabled, exception should be thrown

        self.driver1.strict_mode = True
        expected_msg = ('File content-type could not be guessed and no'
                        ' content_type value is provided')
        self.assertRaisesRegexp(AttributeError, expected_msg,
                                self.driver1._upload_object,
                                object_name='test',
                                content_type=None,
                                upload_func=upload_func,
                                upload_func_kwargs={},
                                request_path='/',
                                iterator=iterator)
Exemplo n.º 12
0
class BaseStorageTests(unittest.TestCase):
    def setUp(self):
        self.send_called = 0
        StorageDriver.connectionCls.conn_class = BaseMockRawResponse

        self.driver1 = StorageDriver('username', 'key', host='localhost')
        self.driver1.supports_chunked_encoding = True

        self.driver2 = StorageDriver('username', 'key', host='localhost')
        self.driver2.supports_chunked_encoding = False

        self.driver1.strict_mode = False
        self.driver1.strict_mode = False

    def test__upload_object_iterator_must_have_next_method(self):

        valid_iterators = [BytesIO(b('134')), StringIO('bar')]
        invalid_iterators = ['foobar', '', False, True, 1, object()]

        def upload_func(*args, **kwargs):
            return True, 'barfoo', 100

        kwargs = {
            'object_name': 'foo',
            'content_type': 'foo/bar',
            'upload_func': upload_func,
            'upload_func_kwargs': {},
            'request_path': '/',
            'headers': {}
        }

        for value in valid_iterators:
            kwargs['stream'] = value
            self.driver1._upload_object(**kwargs)

        for value in invalid_iterators:
            kwargs['stream'] = value

            try:
                self.driver1._upload_object(**kwargs)
            except AttributeError:
                pass
            else:
                self.fail('Exception was not thrown')

    def test__get_hash_function(self):
        self.driver1.hash_type = 'md5'
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        self.driver1.hash_type = 'sha1'
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        try:
            self.driver1.hash_type = 'invalid-hash-function'
            func = self.driver1._get_hash_function()
        except RuntimeError:
            pass
        else:
            self.fail('Invalid hash type but exception was not thrown')

    def test_upload_no_content_type_supplied_or_detected(self):
        iterator = StringIO()

        upload_func = Mock()
        upload_func.return_value = True, '', 0

        # strict_mode is disabled, default content type should be used
        self.driver1.connection = Mock()

        self.driver1._upload_object(object_name='test',
                                    content_type=None,
                                    upload_func=upload_func,
                                    upload_func_kwargs={},
                                    request_path='/',
                                    stream=iterator)

        headers = self.driver1.connection.request.call_args[-1]['headers']
        self.assertEqual(headers['Content-Type'], DEFAULT_CONTENT_TYPE)

        # strict_mode is enabled, exception should be thrown

        self.driver1.strict_mode = True
        expected_msg = ('File content-type could not be guessed and no'
                        ' content_type value is provided')
        self.assertRaisesRegexp(AttributeError,
                                expected_msg,
                                self.driver1._upload_object,
                                object_name='test',
                                content_type=None,
                                upload_func=upload_func,
                                upload_func_kwargs={},
                                request_path='/',
                                stream=iterator)

    @mock.patch('libcloud.utils.files.exhaust_iterator')
    @mock.patch('libcloud.utils.files.read_in_chunks')
    def test_upload_object_hash_calculation_is_efficient(
            self, mock_read_in_chunks, mock_exhaust_iterator):
        # Verify that we don't buffer whole file in memory when calculating
        # object has when iterator has __next__ method, but instead read and calculate hash in chunks
        size = 100

        self.driver1.connection = Mock()

        # stream has __next__ method and next() method
        mock_read_in_chunks.return_value = 'a' * size

        iterator = BodyStream('a' * size)
        self.assertTrue(hasattr(iterator, '__next__'))
        self.assertTrue(hasattr(iterator, 'next'))

        upload_func = Mock()
        upload_func.return_value = True, '', size

        self.assertEqual(mock_read_in_chunks.call_count, 0)
        self.assertEqual(mock_exhaust_iterator.call_count, 0)

        result = self.driver1._upload_object(object_name='test1',
                                             content_type=None,
                                             upload_func=upload_func,
                                             upload_func_kwargs={},
                                             request_path='/',
                                             stream=iterator)

        hasher = hashlib.md5()
        hasher.update(b('a') * size)
        expected_hash = hasher.hexdigest()

        self.assertEqual(result['data_hash'], expected_hash)
        self.assertEqual(result['bytes_transferred'], size)

        headers = self.driver1.connection.request.call_args[-1]['headers']
        self.assertEqual(headers['Content-Type'], DEFAULT_CONTENT_TYPE)

        self.assertEqual(mock_read_in_chunks.call_count, 1)
        self.assertEqual(mock_exhaust_iterator.call_count, 0)

        # stream has only has next() method
        mock_read_in_chunks.return_value = 'b' * size

        iterator = iter([str(v) for v in ['b' * size]])

        if PY2:
            self.assertFalse(hasattr(iterator, '__next__'))
            self.assertTrue(hasattr(iterator, 'next'))
        else:
            self.assertTrue(hasattr(iterator, '__next__'))
            self.assertFalse(hasattr(iterator, 'next'))

        self.assertEqual(mock_read_in_chunks.call_count, 1)
        self.assertEqual(mock_exhaust_iterator.call_count, 0)

        self.assertEqual(mock_read_in_chunks.call_count, 1)
        self.assertEqual(mock_exhaust_iterator.call_count, 0)

        result = self.driver1._upload_object(object_name='test2',
                                             content_type=None,
                                             upload_func=upload_func,
                                             upload_func_kwargs={},
                                             request_path='/',
                                             stream=iterator)

        hasher = hashlib.md5()
        hasher.update(b('b') * size)
        expected_hash = hasher.hexdigest()

        self.assertEqual(result['data_hash'], expected_hash)
        self.assertEqual(result['bytes_transferred'], size)

        headers = self.driver1.connection.request.call_args[-1]['headers']
        self.assertEqual(headers['Content-Type'], DEFAULT_CONTENT_TYPE)

        self.assertEqual(mock_read_in_chunks.call_count, 2)
        self.assertEqual(mock_exhaust_iterator.call_count, 0)
Exemplo n.º 13
0
class BaseStorageTests(unittest.TestCase):
    def setUp(self):
        self.send_called = 0
        StorageDriver.connectionCls.conn_class = BaseMockRawResponse

        self.driver1 = StorageDriver('username', 'key', host='localhost')
        self.driver1.supports_chunked_encoding = True

        self.driver2 = StorageDriver('username', 'key', host='localhost')
        self.driver2.supports_chunked_encoding = False

        self.driver1.strict_mode = False
        self.driver1.strict_mode = False

    def test__upload_object_iterator_must_have_next_method(self):

        valid_iterators = [BytesIO(b('134')), StringIO('bar')]
        invalid_iterators = ['foobar', '', False, True, 1, object()]

        kwargs = {'object_name': 'foo', 'content_type': 'foo/bar',
                  'request_path': '/', 'headers': {}}

        for value in valid_iterators:
            kwargs['stream'] = value
            self.driver1._upload_object(**kwargs)

        for value in invalid_iterators:
            kwargs['stream'] = value

            try:
                self.driver1._upload_object(**kwargs)
            except AttributeError:
                pass
            else:
                self.fail('Exception was not thrown')

    def test__upload_object_does_not_stream_response(self):
        resp = self.driver1._upload_object(
            object_name='foo',
            content_type='foo/bar',
            request_path='/',
            stream=iter(b'foo'))
        mock_response = resp["response"].response._response
        response_streamed = mock_response.request.stream
        assert response_streamed is False

    def test__get_hash_function(self):
        self.driver1.hash_type = 'md5'
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        self.driver1.hash_type = 'sha1'
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        try:
            self.driver1.hash_type = 'invalid-hash-function'
            func = self.driver1._get_hash_function()
        except RuntimeError:
            pass
        else:
            self.fail('Invalid hash type but exception was not thrown')

    def test_upload_no_content_type_supplied_or_detected(self):
        iterator = StringIO()

        # strict_mode is disabled, default content type should be used
        self.driver1.connection = Mock()

        self.driver1._upload_object(object_name='test',
                                    content_type=None,
                                    request_path='/',
                                    stream=iterator)

        headers = self.driver1.connection.request.call_args[-1]['headers']
        self.assertEqual(headers['Content-Type'], DEFAULT_CONTENT_TYPE)

        # strict_mode is enabled, exception should be thrown

        self.driver1.strict_mode = True
        expected_msg = ('File content-type could not be guessed for "test" '
                        'and no content_type value is provided')
        assertRaisesRegex(self, AttributeError, expected_msg,
                          self.driver1._upload_object,
                          object_name='test',
                          content_type=None,
                          request_path='/',
                          stream=iterator)

    @mock.patch('libcloud.utils.files.exhaust_iterator')
    @mock.patch('libcloud.utils.files.read_in_chunks')
    def test_upload_object_hash_calculation_is_efficient(self, mock_read_in_chunks,
                                                         mock_exhaust_iterator):
        # Verify that we don't buffer whole file in memory when calculating
        # object has when iterator has __next__ method, but instead read and
        # calculate hash in chunks
        size = 100

        self.driver1.connection = Mock()

        # stream has __next__ method and next() method
        mock_read_in_chunks.return_value = 'a' * size

        iterator = BodyStream('a' * size)
        self.assertTrue(hasattr(iterator, '__next__'))
        self.assertTrue(hasattr(iterator, 'next'))

        self.assertEqual(mock_read_in_chunks.call_count, 0)
        self.assertEqual(mock_exhaust_iterator.call_count, 0)

        result = self.driver1._upload_object(object_name='test1',
                                             content_type=None,
                                             request_path='/',
                                             stream=iterator)

        hasher = hashlib.md5()
        hasher.update(b('a') * size)
        expected_hash = hasher.hexdigest()

        self.assertEqual(result['data_hash'], expected_hash)
        self.assertEqual(result['bytes_transferred'], size)

        headers = self.driver1.connection.request.call_args[-1]['headers']
        self.assertEqual(headers['Content-Type'], DEFAULT_CONTENT_TYPE)

        self.assertEqual(mock_read_in_chunks.call_count, 1)
        self.assertEqual(mock_exhaust_iterator.call_count, 0)

        # stream has only has next() method
        mock_read_in_chunks.return_value = 'b' * size

        iterator = iter([str(v) for v in ['b' * size]])

        if PY2:
            self.assertFalse(hasattr(iterator, '__next__'))
            self.assertTrue(hasattr(iterator, 'next'))
        else:
            self.assertTrue(hasattr(iterator, '__next__'))
            self.assertFalse(hasattr(iterator, 'next'))

        self.assertEqual(mock_read_in_chunks.call_count, 1)
        self.assertEqual(mock_exhaust_iterator.call_count, 0)

        self.assertEqual(mock_read_in_chunks.call_count, 1)
        self.assertEqual(mock_exhaust_iterator.call_count, 0)

        result = self.driver1._upload_object(object_name='test2',
                                             content_type=None,
                                             request_path='/',
                                             stream=iterator)

        hasher = hashlib.md5()
        hasher.update(b('b') * size)
        expected_hash = hasher.hexdigest()

        self.assertEqual(result['data_hash'], expected_hash)
        self.assertEqual(result['bytes_transferred'], size)

        headers = self.driver1.connection.request.call_args[-1]['headers']
        self.assertEqual(headers['Content-Type'], DEFAULT_CONTENT_TYPE)

        self.assertEqual(mock_read_in_chunks.call_count, 2)
        self.assertEqual(mock_exhaust_iterator.call_count, 0)

    def test_upload_object_via_stream_illegal_seek_errors_are_ignored(self):
        # Illegal seek errors should be ignored
        size = 100

        self.driver1.connection = Mock()

        seek_error = OSError('Illegal seek')
        seek_error.errno = 29
        assert errno.ESPIPE == 29

        iterator = BodyStream('a' * size)
        iterator.seek = mock.Mock(side_effect=seek_error)

        result = self.driver1._upload_object(object_name='test1',
                                             content_type=None,
                                             request_path='/',
                                             stream=iterator)

        hasher = hashlib.md5()
        hasher.update(b('a') * size)
        expected_hash = hasher.hexdigest()

        self.assertEqual(result['data_hash'], expected_hash)
        self.assertEqual(result['bytes_transferred'], size)

        # But others shouldn't
        self.driver1.connection = Mock()

        seek_error = OSError('Other error')
        seek_error.errno = 21

        iterator = BodyStream('b' * size)
        iterator.seek = mock.Mock(side_effect=seek_error)

        self.assertRaisesRegex(OSError, 'Other error',
                               self.driver1._upload_object,
                               object_name='test1',
                               content_type=None,
                               request_path='/',
                               stream=iterator)

    def test_get_standard_range_str(self):
        result = self.driver1._get_standard_range_str(0, 5)
        self.assertEqual(result, 'bytes=0-4')

        result = self.driver1._get_standard_range_str(0)
        self.assertEqual(result, 'bytes=0-')
        result = self.driver1._get_standard_range_str(0, 1)

        self.assertEqual(result, 'bytes=0-0')

        result = self.driver1._get_standard_range_str(200)
        self.assertEqual(result, 'bytes=200-')

        result = self.driver1._get_standard_range_str(10, 200)
        self.assertEqual(result, 'bytes=10-199')

        result = self.driver1._get_standard_range_str(10, 11)
        self.assertEqual(result, 'bytes=10-10')

        result = self.driver1._get_standard_range_str(10, 11, True)
        self.assertEqual(result, 'bytes=10-11')
Exemplo n.º 14
0
class BaseStorageTests(unittest.TestCase):

    def setUp(self):
        self.send_called = 0
        StorageDriver.connectionCls.conn_class = StorageMockHttp
        StorageDriver.connectionCls.rawResponseCls = BaseMockRawResponse

        self.driver1 = StorageDriver('username', 'key', host='localhost')
        self.driver1.supports_chunked_encoding = True

        self.driver2 = StorageDriver('username', 'key', host='localhost')
        self.driver2.supports_chunked_encoding = False

        self.driver1.strict_mode = False
        self.driver1.strict_mode = False

    def test__upload_object_iterator_must_have_next_method(self):

        valid_iterators = [BytesIO(b('134')), StringIO('bar')]
        invalid_iterators = ['foobar', '', False, True, 1, object()]

        def upload_func(*args, **kwargs):
            return True, 'barfoo', 100

        kwargs = {'object_name': 'foo', 'content_type': 'foo/bar',
                  'upload_func': upload_func, 'upload_func_kwargs': {},
                  'request_path': '/', 'headers': {}}

        for value in valid_iterators:
            kwargs['stream'] = value
            self.driver1._upload_object(**kwargs)

        for value in invalid_iterators:
            kwargs['stream'] = value

            try:
                self.driver1._upload_object(**kwargs)
            except AttributeError:
                pass
            else:
                self.fail('Exception was not thrown')

    def test__get_hash_function(self):
        self.driver1.hash_type = 'md5'
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        self.driver1.hash_type = 'sha1'
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        try:
            self.driver1.hash_type = 'invalid-hash-function'
            func = self.driver1._get_hash_function()
        except RuntimeError:
            pass
        else:
            self.fail('Invalid hash type but exception was not thrown')

    def test_upload_no_content_type_supplied_or_detected(self):
        iterator = StringIO()

        upload_func = Mock()
        upload_func.return_value = True, '', 0

        # strict_mode is disabled, default content type should be used
        self.driver1.connection = Mock()

        self.driver1._upload_object(object_name='test',
                                    content_type=None,
                                    upload_func=upload_func,
                                    upload_func_kwargs={},
                                    request_path='/',
                                    stream=iterator)

        headers = self.driver1.connection.request.call_args[-1]['headers']
        self.assertEqual(headers['Content-Type'], DEFAULT_CONTENT_TYPE)

        # strict_mode is enabled, exception should be thrown

        self.driver1.strict_mode = True
        expected_msg = ('File content-type could not be guessed and no'
                        ' content_type value is provided')
        self.assertRaisesRegexp(AttributeError, expected_msg,
                                self.driver1._upload_object,
                                object_name='test',
                                content_type=None,
                                upload_func=upload_func,
                                upload_func_kwargs={},
                                request_path='/',
                                stream=iterator)
class BaseStorageTests(unittest.TestCase):
    def setUp(self):
        self.send_called = 0
        StorageDriver.connectionCls.conn_classes = (None, StorageMockHttp)

        self.driver1 = StorageDriver('username', 'key', host='localhost')
        self.driver1.supports_chunked_encoding = True
        self.driver2 = StorageDriver('username', 'key', host='localhost')
        self.driver2.supports_chunked_encoding = False

    def test__upload_object_iterator_must_have_next_method(self):
        class Iterator(object):
            def next(self):
                pass

        class Iterator2(file):
            def __init__(self):
                pass

        class SomeClass(object):
            pass

        valid_iterators = [Iterator(), Iterator2(), StringIO('bar')]
        invalid_iterators = ['foobar', '', False, True, 1, object()]

        def upload_func(*args, **kwargs):
            return True, 'barfoo', 100

        kwargs = {'object_name': 'foo', 'content_type': 'foo/bar',
                   'upload_func': upload_func, 'upload_func_kwargs': {},
                   'request_path': '/', 'headers': {}}

        for value in valid_iterators:
            kwargs['iterator'] = value
            self.driver1._upload_object(**kwargs)

        for value in invalid_iterators:
            kwargs['iterator'] = value

            try:
                self.driver1._upload_object(**kwargs)
            except AttributeError:
                pass
            else:
                self.fail('Exception was not thrown')

    def test_upload_zero_bytes_long_object_via_stream(self):
        iterator = Mock()

        if PY3:
            iterator.__next__ = Mock()
            iterator.__next__.side_effect = StopIteration()
        else:
            iterator.next.side_effect = StopIteration()

        def mock_send(data):
            self.send_called += 1

        response = Mock()
        response.connection.connection.send = mock_send

        # Normal
        success, data_hash, bytes_transferred = \
                 self.driver1._stream_data(response=response,
                                           iterator=iterator,
                                           chunked=False, calculate_hash=True)

        self.assertTrue(success)
        self.assertEqual(data_hash, hashlib.md5(b('')).hexdigest())
        self.assertEqual(bytes_transferred, 0)
        self.assertEqual(self.send_called, 1)

        # Chunked
        success, data_hash, bytes_transferred = \
                 self.driver1._stream_data(response=response,
                                           iterator=iterator,
                                           chunked=True, calculate_hash=True)

        self.assertTrue(success)
        self.assertEqual(data_hash, hashlib.md5(b('')).hexdigest())
        self.assertEqual(bytes_transferred, 0)
        self.assertEqual(self.send_called, 5)

    def test__upload_data(self):
        def mock_send(data):
            self.send_called += 1

        response = Mock()
        response.connection.connection.send = mock_send

        data = '123456789901234567'
        success, data_hash, bytes_transferred = \
                 self.driver1._upload_data(response=response, data=data,
                                           calculate_hash=True)

        self.assertTrue(success)
        self.assertEqual(data_hash, hashlib.md5(b(data)).hexdigest())
        self.assertEqual(bytes_transferred, (len(data)))
        self.assertEqual(self.send_called, 1)

    def test__get_hash_function(self):
        self.driver1.hash_type = 'md5'
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        self.driver1.hash_type = 'sha1'
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        try:
            self.driver1.hash_type = 'invalid-hash-function'
            func = self.driver1._get_hash_function()
        except RuntimeError:
            pass
        else:
            self.fail('Invalid hash type but exception was not thrown')
Exemplo n.º 16
0
class BaseStorageTests(unittest.TestCase):
    def setUp(self):
        self.send_called = 0
        StorageDriver.connectionCls.conn_class = StorageMockHttp
        StorageDriver.connectionCls.rawResponseCls = BaseMockRawResponse

        self.driver1 = StorageDriver('username', 'key', host='localhost')
        self.driver1.supports_chunked_encoding = True

        self.driver2 = StorageDriver('username', 'key', host='localhost')
        self.driver2.supports_chunked_encoding = False

        self.driver1.strict_mode = False
        self.driver1.strict_mode = False

    def test__upload_object_iterator_must_have_next_method(self):

        valid_iterators = [BytesIO(b('134')), StringIO('bar')]
        invalid_iterators = ['foobar', '', False, True, 1, object()]

        def upload_func(*args, **kwargs):
            return True, 'barfoo', 100

        kwargs = {
            'object_name': 'foo',
            'content_type': 'foo/bar',
            'upload_func': upload_func,
            'upload_func_kwargs': {},
            'request_path': '/',
            'headers': {}
        }

        for value in valid_iterators:
            kwargs['stream'] = value
            self.driver1._upload_object(**kwargs)

        for value in invalid_iterators:
            kwargs['stream'] = value

            try:
                self.driver1._upload_object(**kwargs)
            except AttributeError:
                pass
            else:
                self.fail('Exception was not thrown')

    def test__get_hash_function(self):
        self.driver1.hash_type = 'md5'
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        self.driver1.hash_type = 'sha1'
        func = self.driver1._get_hash_function()
        self.assertTrue(func)

        try:
            self.driver1.hash_type = 'invalid-hash-function'
            func = self.driver1._get_hash_function()
        except RuntimeError:
            pass
        else:
            self.fail('Invalid hash type but exception was not thrown')

    def test_upload_no_content_type_supplied_or_detected(self):
        iterator = StringIO()

        upload_func = Mock()
        upload_func.return_value = True, '', 0

        # strict_mode is disabled, default content type should be used
        self.driver1.connection = Mock()

        self.driver1._upload_object(object_name='test',
                                    content_type=None,
                                    upload_func=upload_func,
                                    upload_func_kwargs={},
                                    request_path='/',
                                    stream=iterator)

        headers = self.driver1.connection.request.call_args[-1]['headers']
        self.assertEqual(headers['Content-Type'], DEFAULT_CONTENT_TYPE)

        # strict_mode is enabled, exception should be thrown

        self.driver1.strict_mode = True
        expected_msg = ('File content-type could not be guessed and no'
                        ' content_type value is provided')
        self.assertRaisesRegexp(AttributeError,
                                expected_msg,
                                self.driver1._upload_object,
                                object_name='test',
                                content_type=None,
                                upload_func=upload_func,
                                upload_func_kwargs={},
                                request_path='/',
                                stream=iterator)