예제 #1
0
 def setUp(self):
     if Application is None:
         raise SkipTest
     self.client = Client(retries=1, swift_proxy=True,
                          swift_proxy_storage_path=SWIFT_PROXY_STORAGE_PATH)
     self.uuid = uuid4().hex
     self.container = 'swiftly_test_' + self.uuid
     status = self.client.put_container(self.container)[0]
     self.assertTrue(status // 100 == 2)
예제 #2
0
 def setUp(self):
     self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY)
     self.uuid = uuid4().hex
     self.container = 'swiftly_test_' + self.uuid
     status = self.client.put_container(self.container)[0]
     self.assertTrue(status // 100 == 2)
     status = \
         self.client.put_object(self.container, 'object1', '123456789')[0]
     self.assertTrue(status // 100 == 2)
예제 #3
0
class TestAccountClient(unittest.TestCase):

    def setUp(self):
        self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY, retries=1)

    def test_head(self):
        status, reason, headers, contents = self.client.head_account()
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertTrue('x-account-bytes-used' in headers)
        self.assertTrue(isinstance(contents, str))

    def test_get(self):
        uuid = uuid4().hex
        try:
            status, reason, headers, contents = \
                self.client.put_container('swiftly_test_' + uuid)
            self.assertTrue(status // 100 == 2)
            status, reason, headers, contents = \
                self.client.get_account(prefix='swiftly_test_' + uuid)
            self.assertTrue(status // 100 == 2)
            self.assertTrue(isinstance(reason, str))
            self.assertTrue(int(headers['x-account-container-count']) > 0)
            self.assertTrue(isinstance(contents, list))
            found = False
            for c in contents:
                if c['name'] == 'swiftly_test_' + uuid:
                    found = True
                    break
            self.assertTrue(found)
        finally:
            self.client.delete_container('swiftly_test_' + uuid)

    def test_post(self):
        uuid = uuid4().hex
        try:
            status, reason, headers, contents = self.client.post_account(
                headers={'x-account-meta-swiftly-test-' + uuid: '123456789'})
            self.assertTrue(status // 100 == 2)
            self.assertTrue(isinstance(reason, str))
            self.assertTrue(isinstance(headers, dict))
            self.assertTrue(isinstance(contents, str))
            status, reason, headers, contents = self.client.head_account()
            self.assertTrue(status // 100 == 2)
            self.assertEquals(headers['x-account-meta-swiftly-test-' + uuid],
                              '123456789')
            status, reason, headers, contents = self.client.post_account(
                headers={'x-account-meta-swiftly-test-' + uuid: ''})
            self.assertTrue(status // 100 == 2)
            status, reason, headers, contents = self.client.head_account()
            self.assertTrue(status // 100 == 2)
            self.assertTrue(
                'x-account-meta-swiftly-test-' + uuid not in headers)
        finally:
            status, reason, headers, contents = self.client.post_account(
                headers={'x-account-meta-swiftly-test-' + uuid: ''})
예제 #4
0
 def test_head_bad_auth(self):
     self.client = Client(AUTH_URL,
                          AUTH_USER,
                          AUTH_KEY + 'invalid',
                          retries=0)
     exc = None
     try:
         self.client.head_account()
     except HTTPException, err:
         exc = err
예제 #5
0
class TestAccountClient(unittest.TestCase):
    def setUp(self):
        self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY, retries=1)

    def test_head(self):
        status, reason, headers, contents = self.client.head_account()
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertTrue('x-account-bytes-used' in headers)
        self.assertTrue(isinstance(contents, str))

    def test_get(self):
        uuid = uuid4().hex
        try:
            status, reason, headers, contents = \
                self.client.put_container('swiftly_test_' + uuid)
            self.assertTrue(status // 100 == 2)
            status, reason, headers, contents = \
                self.client.get_account(prefix='swiftly_test_' + uuid)
            self.assertTrue(status // 100 == 2)
            self.assertTrue(isinstance(reason, str))
            self.assertTrue(int(headers['x-account-container-count']) > 0)
            self.assertTrue(isinstance(contents, list))
            found = False
            for c in contents:
                if c['name'] == 'swiftly_test_' + uuid:
                    found = True
                    break
            self.assertTrue(found)
        finally:
            self.client.delete_container('swiftly_test_' + uuid)

    def test_post(self):
        uuid = uuid4().hex
        try:
            status, reason, headers, contents = self.client.post_account(
                headers={'x-account-meta-swiftly-test-' + uuid: '123456789'})
            self.assertTrue(status // 100 == 2)
            self.assertTrue(isinstance(reason, str))
            self.assertTrue(isinstance(headers, dict))
            self.assertTrue(isinstance(contents, str))
            status, reason, headers, contents = self.client.head_account()
            self.assertTrue(status // 100 == 2)
            self.assertEquals(headers['x-account-meta-swiftly-test-' + uuid],
                              '123456789')
            status, reason, headers, contents = self.client.post_account(
                headers={'x-account-meta-swiftly-test-' + uuid: ''})
            self.assertTrue(status // 100 == 2)
            status, reason, headers, contents = self.client.head_account()
            self.assertTrue(status // 100 == 2)
            self.assertTrue('x-account-meta-swiftly-test-' +
                            uuid not in headers)
        finally:
            status, reason, headers, contents = self.client.post_account(
                headers={'x-account-meta-swiftly-test-' + uuid: ''})
예제 #6
0
class TestObjectDirect(TestObjectClient):

    def setUp(self):
        if Application is None:
            raise SkipTest
        self.client = Client(retries=1, swift_proxy=True,
                             swift_proxy_storage_path=SWIFT_PROXY_STORAGE_PATH)
        self.uuid = uuid4().hex
        self.container = 'swiftly_test_' + self.uuid
        status = self.client.put_container(self.container)[0]
        self.assertTrue(status // 100 == 2)
        status = \
            self.client.put_object(self.container, 'object1', '123456789')[0]
        self.assertTrue(status // 100 == 2)
예제 #7
0
class TestAcountClientOnly(unittest.TestCase):

    def setUp(self):
        self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY, retries=1)

    def test_head_bad_auth(self):
        self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY + 'invalid',
                             retries=0)
        exc = None
        try:
            self.client.head_account()
        except HTTPException, err:
            exc = err
        self.assertTrue(exc is not None)
        self.assertEquals(exc.args[1], 401)
예제 #8
0
class TestAcountClientOnly(unittest.TestCase):
    def setUp(self):
        self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY, retries=1)

    def test_head_bad_auth(self):
        self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY + "invalid", retries=0)
        exc = None
        try:
            self.client.head_account()
        except HTTPException, err:
            exc = err
        self.assertTrue(exc is not None)
        self.assertEquals(
            str(exc), "Auth failure ['401 Unauthorized', '400 Bad Request', " "'400 Bad Request', '400 Bad Request']."
        )
예제 #9
0
 def test_head_bad_auth(self):
     self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY + "invalid", retries=0)
     exc = None
     try:
         self.client.head_account()
     except HTTPException, err:
         exc = err
예제 #10
0
 def setUp(self):
     self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY)
     self.uuid = uuid4().hex
     self.container = 'swiftly_test_' + self.uuid
     status = self.client.put_container(self.container)[0]
     self.assertTrue(status // 100 == 2)
     status = \
         self.client.put_object(self.container, 'object1', '123456789')[0]
     self.assertTrue(status // 100 == 2)
예제 #11
0
class TestAcountClientOnly(unittest.TestCase):
    def setUp(self):
        self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY, retries=1)

    def test_head_bad_auth(self):
        self.client = Client(AUTH_URL,
                             AUTH_USER,
                             AUTH_KEY + 'invalid',
                             retries=0)
        exc = None
        try:
            self.client.head_account()
        except HTTPException, err:
            exc = err
        self.assertTrue(exc is not None)
        self.assertEquals(
            str(exc), "Auth failure ['401 Unauthorized', '400 Bad Request', "
            "'400 Bad Request', '400 Bad Request'].")
예제 #12
0
 def open(self):
     self.client = Client(
         auth_url,
         auth_user,
         auth_key,
         proxy=proxy,
         retries=retries,
         swift_proxy=swift_proxy,
         cache_path=cache_path,
     )
     try:
         (status, reason, headers, contents) = \
             __swiftly_client.head_account()
     except:
         self.logger.error('Connection failed: connection refused.')
         return False
     if status != 200:
         self.logger.error('Connection failed: (%d) %s' % (status, reason))
         return False
     self.logger.debug('Connection made to swift.')
     return True
예제 #13
0
파일: swift.py 프로젝트: vindeka/gate
 def open(self):
     self.client = Client(
         auth_url,
         auth_user,
         auth_key,
         proxy=proxy,
         retries=retries,
         swift_proxy=swift_proxy,
         cache_path=cache_path,
         )
     try:
         (status, reason, headers, contents) = \
             __swiftly_client.head_account()
     except:
         self.logger.error('Connection failed: connection refused.')
         return False
     if status != 200:
         self.logger.error('Connection failed: (%d) %s' % (status,
                           reason))
         return False
     self.logger.debug('Connection made to swift.')
     return True
예제 #14
0
class TestContainerClient(unittest.TestCase):

    def setUp(self):
        self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY)
        self.uuid = uuid4().hex
        self.container = 'swiftly_test_' + self.uuid
        status = self.client.put_container(self.container)[0]
        self.assertTrue(status // 100 == 2)

    def tearDown(self):
        self.client.delete_container(self.container)

    def test_head(self):
        status, reason, headers, contents = \
            self.client.head_container(self.container)
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertTrue('x-container-bytes-used' in headers)
        self.assertTrue(isinstance(contents, str))

    def test_get(self):
        try:
            status, reason, headers, contents = \
                self.client.put_object(self.container, 'object1', 'testvalue')
            self.assertTrue(status // 100 == 2)
            status, reason, headers, contents = \
                self.client.get_container(self.container)
            self.assertTrue(status // 100 == 2)
            self.assertTrue(isinstance(reason, str))
            self.assertEquals(int(headers['x-container-object-count']), 1)
            self.assertTrue(isinstance(contents, list))
            found = False
            for c in contents:
                if c['name'] == 'object1':
                    found = True
                    break
            self.assertTrue(found)
        finally:
            self.client.delete_object(self.container, 'object1')

    def test_put(self):
        status, reason, headers, contents = self.client.put_container(
            self.container,
            headers={'x-container-meta-swiftly-test': '123456789'})
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertTrue(isinstance(headers, dict))
        self.assertTrue(isinstance(contents, str))
        status, reason, headers, contents = \
            self.client.head_container(self.container)
        self.assertTrue(status // 100 == 2)
        self.assertEquals(
            headers['x-container-meta-swiftly-test'], '123456789')
        status, reason, headers, contents = self.client.put_container(
            self.container, headers={'x-container-meta-swiftly-test': ''})
        self.assertTrue(status // 100 == 2)
        status, reason, headers, contents = \
            self.client.head_container(self.container)
        self.assertTrue(status // 100 == 2)
        self.assertTrue('x-container-meta-swiftly-test' not in headers)

    def test_post(self):
        status, reason, headers, contents = self.client.post_container(
            self.container,
            headers={'x-container-meta-swiftly-test': '123456789'})
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertTrue(isinstance(headers, dict))
        self.assertTrue(isinstance(contents, str))
        status, reason, headers, contents = \
            self.client.head_container(self.container)
        self.assertTrue(status // 100 == 2)
        self.assertEquals(
            headers['x-container-meta-swiftly-test'], '123456789')
        status, reason, headers, contents = self.client.post_container(
            self.container, headers={'x-container-meta-swiftly-test': ''})
        self.assertTrue(status // 100 == 2)
        status, reason, headers, contents = \
            self.client.head_container(self.container)
        self.assertTrue(status // 100 == 2)
        self.assertTrue('x-container-meta-swiftly-test' not in headers)

    def test_delete(self):
        status = self.client.head_container(self.container)[0]
        self.assertTrue(status // 100 == 2)
        status, reason, headers, contents = \
            self.client.delete_container(self.container)
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertTrue(isinstance(headers, dict))
        self.assertTrue(isinstance(contents, str))
        status = self.client.head_container(self.container)[0]
        self.assertEquals(status, 404)
예제 #15
0
class SwiftTransport(object):

    name = 'swift'

    def __init__(self, conf):
        self.conf = conf
        self.logger = get_logger(conf, log_route='transport.memcached')
        self.auth_url = conf.get('auth_url', 'http://localhost:5000/')
        self.auth_user = conf.get('auth_user', 'gate')
        self.auth_key = conf.get('auth_key', 'gate')
        self.proxy = conf.get('proxy', 'None')
        if self.proxy == 'None':
            self.proxy = None
        self.retries = int(conf.get('retries', '4'))
        self.swift_proxy = conf.get('swift_proxy', 'None')
        if self.swift_proxy == 'None':
            self.swift_proxy = None
        self.cache_path = conf.get('cache_path',
                                   '/var/gate/object-store/cache')
        dir_path = os.path.dirname(os.path.abspath(cache_path))
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)
        self.concurrency = int(conf.get('put_threads', 5))
        self.segment_size = int(conf.get('segment_size', 262144000))

    def open(self):
        self.client = Client(
            auth_url,
            auth_user,
            auth_key,
            proxy=proxy,
            retries=retries,
            swift_proxy=swift_proxy,
            cache_path=cache_path,
        )
        try:
            (status, reason, headers, contents) = \
                __swiftly_client.head_account()
        except:
            self.logger.error('Connection failed: connection refused.')
            return False
        if status != 200:
            self.logger.error('Connection failed: (%d) %s' % (status, reason))
            return False
        self.logger.debug('Connection made to swift.')
        return True

    def close(self):
        self.client.close()
        self.logger.debug('Connection closed.')

    def get_stream(self, key):
        (container, obj) = key.split('/', limit=1)
        return self.client.get_object(container, obj, stream=True)

    def put_stream(self, key, stream):
        (container, obj) = key.split('/', limit=1)
        size = None
        func_size = getattr(stream, 'size', None)
        if func_size:
            size = func_size()
        if not size:
            return self.client.client.put_object(container, obj, stream)
        if size <= self.segment_size:
            headers = {'Content-Length': size}
            (status, reason, headers, contents) = \
                self.client.put_object(container, obj, stream,
                    headers=headers)
            if status != 201:
                self.logger.error('Failed to put object: %s/%s' %
                                  (container, obj))
                return False
        cont_prefix = '%s_segments' % container
        prefix = '%s/%s/' % (obj, self.segment_size)
        conc = Concurrency(self.concurrency)
        start = 0
        segment = 0
        while start < size:
            obj_path = '%s%08d' % (prefix, segment)
            conc.spawn(
                segment,
                self._put_recursive,
                cont_prefix,
                obj_path,
                stream.copy(),
                start,
            )
            for rv in conc.get_results().values():
                if not rv:
                    conc.join()
                    self.logger.error(
                        'Failed to create segments for object: %s/%s' %
                        (container, obj))
                    return rv
            segment += 1
            start += self.client.segment_size
        conc.join()
        for rv in conc.get_results().values():
            if not rv:
                conc.join()
                self.logger.error(
                    'Failed to create segments for object: %s/%s' %
                    (container, obj))
                return rv
        headers = {'Content-Length': 0, 'X-Object-Manifest': '%s/%s' \
                   % (cont_prefix, prefix)}
        (status, reason, headers, contents) = \
            self.client.put_object(container, obj, '', headers=headers)
        if status != 201:
            self.logger.error('Failed put manifest object: %s/%s' %
                              (container, obj))
            return False
        return True

    def _put_recursive(
        container,
        obj,
        stream,
        offset,
        size,
    ):
        stream.seek(offset)
        headers = {'Content-Length': size}
        try:
            (status, reason, headers, contents) = \
                self.client.put_object(container, obj, stream,
                    headers=headers)
            if status != 201:
                return False
        except:
            return False
        return True
예제 #16
0
 def setUp(self):
     if Application is None:
         raise SkipTest
     self.client = Client(retries=1,
                          swift_proxy=True,
                          swift_proxy_storage_path=SWIFT_PROXY_STORAGE_PATH)
예제 #17
0
 def setUp(self):
     self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY, retries=1)
예제 #18
0
class TestObjectClient(unittest.TestCase):
    def setUp(self):
        self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY)
        self.uuid = uuid4().hex
        self.container = 'swiftly_test_' + self.uuid
        status = self.client.put_container(self.container)[0]
        self.assertTrue(status // 100 == 2)
        status = \
            self.client.put_object(self.container, 'object1', '123456789')[0]
        self.assertTrue(status // 100 == 2)

    def tearDown(self):
        self.client.delete_object(self.container, 'object1')
        self.client.delete_container(self.container)

    def test_head(self):
        status, reason, headers, contents = \
            self.client.head_object(self.container, 'object1')
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertEquals(headers['etag'], '25f9e794323b453885f5181f1b624d0b')
        self.assertTrue(isinstance(contents, str))

    def test_get(self):
        status, reason, headers, contents = \
            self.client.get_object(self.container, 'object1')
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertEquals(headers['etag'], '25f9e794323b453885f5181f1b624d0b')
        self.assertTrue(contents.read(), '123456789')

    def test_get_no_stream(self):
        status, reason, headers, contents = \
            self.client.get_object(self.container, 'object1', stream=False)
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertEquals(headers['etag'], '25f9e794323b453885f5181f1b624d0b')
        self.assertTrue(contents, '123456789')

    def test_put(self):
        try:
            status, reason, headers, contents = self.client.put_object(
                self.container,
                'object2',
                'contentvalue',
                headers={'x-object-meta-swiftly-test': 'somevalue'})
            self.assertTrue(status // 100 == 2)
            self.assertTrue(isinstance(reason, str))
            self.assertTrue(isinstance(headers, dict))
            self.assertTrue(isinstance(contents, str))
            status, reason, headers, contents = \
                self.client.get_object(self.container, 'object2')
            self.assertTrue(status // 100 == 2)
            self.assertEquals(headers['etag'],
                              '11a49846cb4877d7f9bb9471b646464f')
            self.assertEquals(headers['x-object-meta-swiftly-test'],
                              'somevalue')
            self.assertEquals(contents.read(), 'contentvalue')
        finally:
            self.client.delete_object(self.container, 'object2')

    def test_put_with_etag(self):
        try:
            status, reason, headers, contents = self.client.put_object(
                self.container,
                'object2',
                'contentvalue',
                headers={
                    'x-object-meta-swiftly-test': 'somevalue',
                    'etag': '11111111111111111111111111111111'
                })
            self.assertEquals(status, 422)
            status, reason, headers, contents = self.client.put_object(
                self.container,
                'object2',
                'contentvalue',
                headers={
                    'x-object-meta-swiftly-test': 'somevalue',
                    'etag': '11a49846cb4877d7f9bb9471b646464f'
                })
            self.assertTrue(status // 100 == 2)
            self.assertTrue(isinstance(reason, str))
            self.assertTrue(isinstance(headers, dict))
            self.assertTrue(isinstance(contents, str))
            status, reason, headers, contents = \
                self.client.get_object(self.container, 'object2')
            self.assertTrue(status // 100 == 2)
            self.assertEquals(headers['etag'],
                              '11a49846cb4877d7f9bb9471b646464f')
            self.assertEquals(headers['x-object-meta-swiftly-test'],
                              'somevalue')
            self.assertEquals(contents.read(), 'contentvalue')
        finally:
            self.client.delete_object(self.container, 'object2')

    def test_put_as_stream(self):
        try:
            status, reason, headers, contents = self.client.put_object(
                self.container,
                'object2',
                StringIO('contentvalue'),
                headers={
                    'content-length': '12',
                    'x-object-meta-swiftly-test': 'somevalue'
                })
            self.assertTrue(status // 100 == 2)
            self.assertTrue(isinstance(reason, str))
            self.assertTrue(isinstance(headers, dict))
            self.assertTrue(isinstance(contents, str))
            status, reason, headers, contents = \
                self.client.get_object(self.container, 'object2')
            self.assertTrue(status // 100 == 2)
            self.assertEquals(headers['etag'],
                              '11a49846cb4877d7f9bb9471b646464f')
            self.assertEquals(headers['x-object-meta-swiftly-test'],
                              'somevalue')
            self.assertEquals(contents.read(), 'contentvalue')
        finally:
            self.client.delete_object(self.container, 'object2')

    def test_put_as_stream_chunked(self):
        try:
            status, reason, headers, contents = self.client.put_object(
                self.container,
                'object2',
                StringIO('contentvalue'),
                headers={'x-object-meta-swiftly-test': 'somevalue'})
            self.assertTrue(status // 100 == 2)
            self.assertTrue(isinstance(reason, str))
            self.assertTrue(isinstance(headers, dict))
            self.assertTrue(isinstance(contents, str))
            status, reason, headers, contents = \
                self.client.get_object(self.container, 'object2')
            self.assertTrue(status // 100 == 2)
            self.assertEquals(headers['etag'],
                              '11a49846cb4877d7f9bb9471b646464f')
            self.assertEquals(headers['x-object-meta-swiftly-test'],
                              'somevalue')
            self.assertEquals(contents.read(), 'contentvalue')
        finally:
            self.client.delete_object(self.container, 'object2')

    def test_post(self):
        status, reason, headers, contents = \
            self.client.head_object(self.container, 'object1')
        self.assertTrue(status // 100 == 2)
        self.assertTrue('x-object-meta-swiftly-test' not in headers)
        status, reason, headers, contents = self.client.post_object(
            self.container,
            'object1',
            headers={'x-object-meta-swiftly-test': 'testvalue'})
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertTrue(isinstance(headers, dict))
        self.assertTrue(isinstance(contents, str))
        status, reason, headers, contents = \
            self.client.head_object(self.container, 'object1')
        self.assertTrue(status // 100 == 2)
        self.assertEquals(headers['x-object-meta-swiftly-test'], 'testvalue')

    def test_delete(self):
        status = self.client.head_object(self.container, 'object1')[0]
        self.assertTrue(status // 100 == 2)
        status, reason, headers, contents = \
            self.client.delete_object(self.container, 'object1')
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertTrue(isinstance(headers, dict))
        self.assertTrue(isinstance(contents, str))
        status = self.client.head_object(self.container, 'object1')[0]
        self.assertEquals(status, 404)
예제 #19
0
class TestObjectClient(unittest.TestCase):

    def setUp(self):
        self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY)
        self.uuid = uuid4().hex
        self.container = 'swiftly_test_' + self.uuid
        status = self.client.put_container(self.container)[0]
        self.assertTrue(status // 100 == 2)
        status = \
            self.client.put_object(self.container, 'object1', '123456789')[0]
        self.assertTrue(status // 100 == 2)

    def tearDown(self):
        self.client.delete_object(self.container, 'object1')
        self.client.delete_container(self.container)

    def test_head(self):
        status, reason, headers, contents = \
            self.client.head_object(self.container, 'object1')
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertEquals(headers['etag'], '25f9e794323b453885f5181f1b624d0b')
        self.assertTrue(isinstance(contents, str))

    def test_get(self):
        status, reason, headers, contents = \
            self.client.get_object(self.container, 'object1')
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertEquals(headers['etag'], '25f9e794323b453885f5181f1b624d0b')
        self.assertTrue(contents.read(), '123456789')

    def test_get_no_stream(self):
        status, reason, headers, contents = \
            self.client.get_object(self.container, 'object1', stream=False)
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertEquals(headers['etag'], '25f9e794323b453885f5181f1b624d0b')
        self.assertTrue(contents, '123456789')

    def test_put(self):
        try:
            status, reason, headers, contents = self.client.put_object(
                self.container, 'object2', 'contentvalue',
                headers={'x-object-meta-swiftly-test': 'somevalue'})
            self.assertTrue(status // 100 == 2)
            self.assertTrue(isinstance(reason, str))
            self.assertTrue(isinstance(headers, dict))
            self.assertTrue(isinstance(contents, str))
            status, reason, headers, contents = \
                self.client.get_object(self.container, 'object2')
            self.assertTrue(status // 100 == 2)
            self.assertEquals(
                headers['etag'], '11a49846cb4877d7f9bb9471b646464f')
            self.assertEquals(
                headers['x-object-meta-swiftly-test'], 'somevalue')
            self.assertEquals(contents.read(), 'contentvalue')
        finally:
            self.client.delete_object(self.container, 'object2')

    def test_put_with_etag(self):
        try:
            status, reason, headers, contents = self.client.put_object(
                self.container, 'object2', 'contentvalue',
                headers={'x-object-meta-swiftly-test': 'somevalue',
                         'etag': '11111111111111111111111111111111'})
            self.assertEquals(status, 422)
            status, reason, headers, contents = self.client.put_object(
                self.container, 'object2', 'contentvalue',
                headers={'x-object-meta-swiftly-test': 'somevalue',
                         'etag': '11a49846cb4877d7f9bb9471b646464f'})
            self.assertTrue(status // 100 == 2)
            self.assertTrue(isinstance(reason, str))
            self.assertTrue(isinstance(headers, dict))
            self.assertTrue(isinstance(contents, str))
            status, reason, headers, contents = \
                self.client.get_object(self.container, 'object2')
            self.assertTrue(status // 100 == 2)
            self.assertEquals(
                headers['etag'], '11a49846cb4877d7f9bb9471b646464f')
            self.assertEquals(
                headers['x-object-meta-swiftly-test'], 'somevalue')
            self.assertEquals(contents.read(), 'contentvalue')
        finally:
            self.client.delete_object(self.container, 'object2')

    def test_put_as_stream(self):
        try:
            status, reason, headers, contents = self.client.put_object(
                self.container, 'object2', StringIO('contentvalue'),
                headers={'content-length': '12',
                         'x-object-meta-swiftly-test': 'somevalue'})
            self.assertTrue(status // 100 == 2)
            self.assertTrue(isinstance(reason, str))
            self.assertTrue(isinstance(headers, dict))
            self.assertTrue(isinstance(contents, str))
            status, reason, headers, contents = \
                self.client.get_object(self.container, 'object2')
            self.assertTrue(status // 100 == 2)
            self.assertEquals(
                headers['etag'], '11a49846cb4877d7f9bb9471b646464f')
            self.assertEquals(
                headers['x-object-meta-swiftly-test'], 'somevalue')
            self.assertEquals(contents.read(), 'contentvalue')
        finally:
            self.client.delete_object(self.container, 'object2')

    def test_put_as_stream_chunked(self):
        try:
            status, reason, headers, contents = self.client.put_object(
                self.container, 'object2', StringIO('contentvalue'),
                headers={'x-object-meta-swiftly-test': 'somevalue'})
            self.assertTrue(status // 100 == 2)
            self.assertTrue(isinstance(reason, str))
            self.assertTrue(isinstance(headers, dict))
            self.assertTrue(isinstance(contents, str))
            status, reason, headers, contents = \
                self.client.get_object(self.container, 'object2')
            self.assertTrue(status // 100 == 2)
            self.assertEquals(
                headers['etag'], '11a49846cb4877d7f9bb9471b646464f')
            self.assertEquals(
                headers['x-object-meta-swiftly-test'], 'somevalue')
            self.assertEquals(contents.read(), 'contentvalue')
        finally:
            self.client.delete_object(self.container, 'object2')

    def test_post(self):
        status, reason, headers, contents = \
            self.client.head_object(self.container, 'object1')
        self.assertTrue(status // 100 == 2)
        self.assertTrue('x-object-meta-swiftly-test' not in headers)
        status, reason, headers, contents = self.client.post_object(
            self.container, 'object1',
            headers={'x-object-meta-swiftly-test': 'testvalue'})
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertTrue(isinstance(headers, dict))
        self.assertTrue(isinstance(contents, str))
        status, reason, headers, contents = \
            self.client.head_object(self.container, 'object1')
        self.assertTrue(status // 100 == 2)
        self.assertEquals(headers['x-object-meta-swiftly-test'], 'testvalue')

    def test_delete(self):
        status = self.client.head_object(self.container, 'object1')[0]
        self.assertTrue(status // 100 == 2)
        status, reason, headers, contents = \
            self.client.delete_object(self.container, 'object1')
        self.assertTrue(status // 100 == 2)
        self.assertTrue(isinstance(reason, str))
        self.assertTrue(isinstance(headers, dict))
        self.assertTrue(isinstance(contents, str))
        status = self.client.head_object(self.container, 'object1')[0]
        self.assertEquals(status, 404)
예제 #20
0
 def setUp(self):
     self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY, retries=1)
예제 #21
0
파일: swift.py 프로젝트: vindeka/gate
class SwiftTransport(object):

    name = 'swift'

    def __init__(self, conf):
        self.conf = conf
        self.logger = get_logger(conf, log_route='transport.memcached')
        self.auth_url = conf.get('auth_url', 'http://localhost:5000/')
        self.auth_user = conf.get('auth_user', 'gate')
        self.auth_key = conf.get('auth_key', 'gate')
        self.proxy = conf.get('proxy', 'None')
        if self.proxy == 'None':
            self.proxy = None
        self.retries = int(conf.get('retries', '4'))
        self.swift_proxy = conf.get('swift_proxy', 'None')
        if self.swift_proxy == 'None':
            self.swift_proxy = None
        self.cache_path = conf.get('cache_path',
                                   '/var/gate/object-store/cache')
        dir_path = os.path.dirname(os.path.abspath(cache_path))
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)
        self.concurrency = int(conf.get('put_threads', 5))
        self.segment_size = int(conf.get('segment_size', 262144000))

    def open(self):
        self.client = Client(
            auth_url,
            auth_user,
            auth_key,
            proxy=proxy,
            retries=retries,
            swift_proxy=swift_proxy,
            cache_path=cache_path,
            )
        try:
            (status, reason, headers, contents) = \
                __swiftly_client.head_account()
        except:
            self.logger.error('Connection failed: connection refused.')
            return False
        if status != 200:
            self.logger.error('Connection failed: (%d) %s' % (status,
                              reason))
            return False
        self.logger.debug('Connection made to swift.')
        return True

    def close(self):
        self.client.close()
        self.logger.debug('Connection closed.')

    def get_stream(self, key):
        (container, obj) = key.split('/', limit=1)
        return self.client.get_object(container, obj, stream=True)

    def put_stream(self, key, stream):
        (container, obj) = key.split('/', limit=1)
        size = None
        func_size = getattr(stream, 'size', None)
        if func_size:
            size = func_size()
        if not size:
            return self.client.client.put_object(container, obj, stream)
        if size <= self.segment_size:
            headers = {'Content-Length': size}
            (status, reason, headers, contents) = \
                self.client.put_object(container, obj, stream,
                    headers=headers)
            if status != 201:
                self.logger.error('Failed to put object: %s/%s'
                                  % (container, obj))
                return False
        cont_prefix = '%s_segments' % container
        prefix = '%s/%s/' % (obj, self.segment_size)
        conc = Concurrency(self.concurrency)
        start = 0
        segment = 0
        while start < size:
            obj_path = '%s%08d' % (prefix, segment)
            conc.spawn(
                segment,
                self._put_recursive,
                cont_prefix,
                obj_path,
                stream.copy(),
                start,
                )
            for rv in conc.get_results().values():
                if not rv:
                    conc.join()
                    self.logger.error('Failed to create segments for object: %s/%s'
                             % (container, obj))
                    return rv
            segment += 1
            start += self.client.segment_size
        conc.join()
        for rv in conc.get_results().values():
            if not rv:
                conc.join()
                self.logger.error('Failed to create segments for object: %s/%s'
                                   % (container, obj))
                return rv
        headers = {'Content-Length': 0, 'X-Object-Manifest': '%s/%s' \
                   % (cont_prefix, prefix)}
        (status, reason, headers, contents) = \
            self.client.put_object(container, obj, '', headers=headers)
        if status != 201:
            self.logger.error('Failed put manifest object: %s/%s'
                              % (container, obj))
            return False
        return True

    def _put_recursive(
        container,
        obj,
        stream,
        offset,
        size,
        ):
        stream.seek(offset)
        headers = {'Content-Length': size}
        try:
            (status, reason, headers, contents) = \
                self.client.put_object(container, obj, stream,
                    headers=headers)
            if status != 201:
                return False
        except:
            return False
        return True