def setUp(self):
if Application is None:
raise SkipTest
self.client = Client(retries=1, swift_proxy=True,
swift_proxy_storage_path=SWIFT_PROXY_STORAGE_PATH)
self.uuid = uuid4().hex
self.container = 'swiftly_test_' + self.uuid
status = self.client.put_container(self.container)[0]
self.assertTrue(status // 100 == 2)
def setUp(self):
self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY)
self.uuid = uuid4().hex
self.container = 'swiftly_test_' + self.uuid
status = self.client.put_container(self.container)[0]
self.assertTrue(status // 100 == 2)
status = \
self.client.put_object(self.container, 'object1', '123456789')[0]
self.assertTrue(status // 100 == 2)
class TestAccountClient(unittest.TestCase):
def setUp(self):
self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY, retries=1)
def test_head(self):
status, reason, headers, contents = self.client.head_account()
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue('x-account-bytes-used' in headers)
self.assertTrue(isinstance(contents, str))
def test_get(self):
uuid = uuid4().hex
try:
status, reason, headers, contents = \
self.client.put_container('swiftly_test_' + uuid)
self.assertTrue(status // 100 == 2)
status, reason, headers, contents = \
self.client.get_account(prefix='swiftly_test_' + uuid)
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(int(headers['x-account-container-count']) > 0)
self.assertTrue(isinstance(contents, list))
found = False
for c in contents:
if c['name'] == 'swiftly_test_' + uuid:
found = True
break
self.assertTrue(found)
finally:
self.client.delete_container('swiftly_test_' + uuid)
def test_post(self):
uuid = uuid4().hex
try:
status, reason, headers, contents = self.client.post_account(
headers={'x-account-meta-swiftly-test-' + uuid: '123456789'})
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status, reason, headers, contents = self.client.head_account()
self.assertTrue(status // 100 == 2)
self.assertEquals(headers['x-account-meta-swiftly-test-' + uuid],
'123456789')
status, reason, headers, contents = self.client.post_account(
headers={'x-account-meta-swiftly-test-' + uuid: ''})
self.assertTrue(status // 100 == 2)
status, reason, headers, contents = self.client.head_account()
self.assertTrue(status // 100 == 2)
self.assertTrue(
'x-account-meta-swiftly-test-' + uuid not in headers)
finally:
status, reason, headers, contents = self.client.post_account(
headers={'x-account-meta-swiftly-test-' + uuid: ''})
def test_head_bad_auth(self):
self.client = Client(AUTH_URL,
AUTH_USER,
AUTH_KEY + 'invalid',
retries=0)
exc = None
try:
self.client.head_account()
except HTTPException, err:
exc = err
class TestAccountClient(unittest.TestCase):
def setUp(self):
self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY, retries=1)
def test_head(self):
status, reason, headers, contents = self.client.head_account()
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue('x-account-bytes-used' in headers)
self.assertTrue(isinstance(contents, str))
def test_get(self):
uuid = uuid4().hex
try:
status, reason, headers, contents = \
self.client.put_container('swiftly_test_' + uuid)
self.assertTrue(status // 100 == 2)
status, reason, headers, contents = \
self.client.get_account(prefix='swiftly_test_' + uuid)
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(int(headers['x-account-container-count']) > 0)
self.assertTrue(isinstance(contents, list))
found = False
for c in contents:
if c['name'] == 'swiftly_test_' + uuid:
found = True
break
self.assertTrue(found)
finally:
self.client.delete_container('swiftly_test_' + uuid)
def test_post(self):
uuid = uuid4().hex
try:
status, reason, headers, contents = self.client.post_account(
headers={'x-account-meta-swiftly-test-' + uuid: '123456789'})
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status, reason, headers, contents = self.client.head_account()
self.assertTrue(status // 100 == 2)
self.assertEquals(headers['x-account-meta-swiftly-test-' + uuid],
'123456789')
status, reason, headers, contents = self.client.post_account(
headers={'x-account-meta-swiftly-test-' + uuid: ''})
self.assertTrue(status // 100 == 2)
status, reason, headers, contents = self.client.head_account()
self.assertTrue(status // 100 == 2)
self.assertTrue('x-account-meta-swiftly-test-' +
uuid not in headers)
finally:
status, reason, headers, contents = self.client.post_account(
headers={'x-account-meta-swiftly-test-' + uuid: ''})
class TestObjectDirect(TestObjectClient):
def setUp(self):
if Application is None:
raise SkipTest
self.client = Client(retries=1, swift_proxy=True,
swift_proxy_storage_path=SWIFT_PROXY_STORAGE_PATH)
self.uuid = uuid4().hex
self.container = 'swiftly_test_' + self.uuid
status = self.client.put_container(self.container)[0]
self.assertTrue(status // 100 == 2)
status = \
self.client.put_object(self.container, 'object1', '123456789')[0]
self.assertTrue(status // 100 == 2)
class TestAcountClientOnly(unittest.TestCase):
def setUp(self):
self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY, retries=1)
def test_head_bad_auth(self):
self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY + 'invalid',
retries=0)
exc = None
try:
self.client.head_account()
except HTTPException, err:
exc = err
self.assertTrue(exc is not None)
self.assertEquals(exc.args[1], 401)
class TestAcountClientOnly(unittest.TestCase):
def setUp(self):
self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY, retries=1)
def test_head_bad_auth(self):
self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY + "invalid", retries=0)
exc = None
try:
self.client.head_account()
except HTTPException, err:
exc = err
self.assertTrue(exc is not None)
self.assertEquals(
str(exc), "Auth failure ['401 Unauthorized', '400 Bad Request', " "'400 Bad Request', '400 Bad Request']."
)
def test_head_bad_auth(self):
self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY + "invalid", retries=0)
exc = None
try:
self.client.head_account()
except HTTPException, err:
exc = err
def setUp(self):
self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY)
self.uuid = uuid4().hex
self.container = 'swiftly_test_' + self.uuid
status = self.client.put_container(self.container)[0]
self.assertTrue(status // 100 == 2)
status = \
self.client.put_object(self.container, 'object1', '123456789')[0]
self.assertTrue(status // 100 == 2)
class TestAcountClientOnly(unittest.TestCase):
def setUp(self):
self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY, retries=1)
def test_head_bad_auth(self):
self.client = Client(AUTH_URL,
AUTH_USER,
AUTH_KEY + 'invalid',
retries=0)
exc = None
try:
self.client.head_account()
except HTTPException, err:
exc = err
self.assertTrue(exc is not None)
self.assertEquals(
str(exc), "Auth failure ['401 Unauthorized', '400 Bad Request', "
"'400 Bad Request', '400 Bad Request'].")
def open(self):
self.client = Client(
auth_url,
auth_user,
auth_key,
proxy=proxy,
retries=retries,
swift_proxy=swift_proxy,
cache_path=cache_path,
)
try:
(status, reason, headers, contents) = \
__swiftly_client.head_account()
except:
self.logger.error('Connection failed: connection refused.')
return False
if status != 200:
self.logger.error('Connection failed: (%d) %s' % (status, reason))
return False
self.logger.debug('Connection made to swift.')
return True
def open(self):
self.client = Client(
auth_url,
auth_user,
auth_key,
proxy=proxy,
retries=retries,
swift_proxy=swift_proxy,
cache_path=cache_path,
)
try:
(status, reason, headers, contents) = \
__swiftly_client.head_account()
except:
self.logger.error('Connection failed: connection refused.')
return False
if status != 200:
self.logger.error('Connection failed: (%d) %s' % (status,
reason))
return False
self.logger.debug('Connection made to swift.')
return True
class TestContainerClient(unittest.TestCase):
def setUp(self):
self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY)
self.uuid = uuid4().hex
self.container = 'swiftly_test_' + self.uuid
status = self.client.put_container(self.container)[0]
self.assertTrue(status // 100 == 2)
def tearDown(self):
self.client.delete_container(self.container)
def test_head(self):
status, reason, headers, contents = \
self.client.head_container(self.container)
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue('x-container-bytes-used' in headers)
self.assertTrue(isinstance(contents, str))
def test_get(self):
try:
status, reason, headers, contents = \
self.client.put_object(self.container, 'object1', 'testvalue')
self.assertTrue(status // 100 == 2)
status, reason, headers, contents = \
self.client.get_container(self.container)
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertEquals(int(headers['x-container-object-count']), 1)
self.assertTrue(isinstance(contents, list))
found = False
for c in contents:
if c['name'] == 'object1':
found = True
break
self.assertTrue(found)
finally:
self.client.delete_object(self.container, 'object1')
def test_put(self):
status, reason, headers, contents = self.client.put_container(
self.container,
headers={'x-container-meta-swiftly-test': '123456789'})
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status, reason, headers, contents = \
self.client.head_container(self.container)
self.assertTrue(status // 100 == 2)
self.assertEquals(
headers['x-container-meta-swiftly-test'], '123456789')
status, reason, headers, contents = self.client.put_container(
self.container, headers={'x-container-meta-swiftly-test': ''})
self.assertTrue(status // 100 == 2)
status, reason, headers, contents = \
self.client.head_container(self.container)
self.assertTrue(status // 100 == 2)
self.assertTrue('x-container-meta-swiftly-test' not in headers)
def test_post(self):
status, reason, headers, contents = self.client.post_container(
self.container,
headers={'x-container-meta-swiftly-test': '123456789'})
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status, reason, headers, contents = \
self.client.head_container(self.container)
self.assertTrue(status // 100 == 2)
self.assertEquals(
headers['x-container-meta-swiftly-test'], '123456789')
status, reason, headers, contents = self.client.post_container(
self.container, headers={'x-container-meta-swiftly-test': ''})
self.assertTrue(status // 100 == 2)
status, reason, headers, contents = \
self.client.head_container(self.container)
self.assertTrue(status // 100 == 2)
self.assertTrue('x-container-meta-swiftly-test' not in headers)
def test_delete(self):
status = self.client.head_container(self.container)[0]
self.assertTrue(status // 100 == 2)
status, reason, headers, contents = \
self.client.delete_container(self.container)
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status = self.client.head_container(self.container)[0]
self.assertEquals(status, 404)
class SwiftTransport(object):
name = 'swift'
def __init__(self, conf):
self.conf = conf
self.logger = get_logger(conf, log_route='transport.memcached')
self.auth_url = conf.get('auth_url', 'http://localhost:5000/')
self.auth_user = conf.get('auth_user', 'gate')
self.auth_key = conf.get('auth_key', 'gate')
self.proxy = conf.get('proxy', 'None')
if self.proxy == 'None':
self.proxy = None
self.retries = int(conf.get('retries', '4'))
self.swift_proxy = conf.get('swift_proxy', 'None')
if self.swift_proxy == 'None':
self.swift_proxy = None
self.cache_path = conf.get('cache_path',
'/var/gate/object-store/cache')
dir_path = os.path.dirname(os.path.abspath(cache_path))
if not os.path.exists(dir_path):
os.makedirs(dir_path)
self.concurrency = int(conf.get('put_threads', 5))
self.segment_size = int(conf.get('segment_size', 262144000))
def open(self):
self.client = Client(
auth_url,
auth_user,
auth_key,
proxy=proxy,
retries=retries,
swift_proxy=swift_proxy,
cache_path=cache_path,
)
try:
(status, reason, headers, contents) = \
__swiftly_client.head_account()
except:
self.logger.error('Connection failed: connection refused.')
return False
if status != 200:
self.logger.error('Connection failed: (%d) %s' % (status, reason))
return False
self.logger.debug('Connection made to swift.')
return True
def close(self):
self.client.close()
self.logger.debug('Connection closed.')
def get_stream(self, key):
(container, obj) = key.split('/', limit=1)
return self.client.get_object(container, obj, stream=True)
def put_stream(self, key, stream):
(container, obj) = key.split('/', limit=1)
size = None
func_size = getattr(stream, 'size', None)
if func_size:
size = func_size()
if not size:
return self.client.client.put_object(container, obj, stream)
if size <= self.segment_size:
headers = {'Content-Length': size}
(status, reason, headers, contents) = \
self.client.put_object(container, obj, stream,
headers=headers)
if status != 201:
self.logger.error('Failed to put object: %s/%s' %
(container, obj))
return False
cont_prefix = '%s_segments' % container
prefix = '%s/%s/' % (obj, self.segment_size)
conc = Concurrency(self.concurrency)
start = 0
segment = 0
while start < size:
obj_path = '%s%08d' % (prefix, segment)
conc.spawn(
segment,
self._put_recursive,
cont_prefix,
obj_path,
stream.copy(),
start,
)
for rv in conc.get_results().values():
if not rv:
conc.join()
self.logger.error(
'Failed to create segments for object: %s/%s' %
(container, obj))
return rv
segment += 1
start += self.client.segment_size
conc.join()
for rv in conc.get_results().values():
if not rv:
conc.join()
self.logger.error(
'Failed to create segments for object: %s/%s' %
(container, obj))
return rv
headers = {'Content-Length': 0, 'X-Object-Manifest': '%s/%s' \
% (cont_prefix, prefix)}
(status, reason, headers, contents) = \
self.client.put_object(container, obj, '', headers=headers)
if status != 201:
self.logger.error('Failed put manifest object: %s/%s' %
(container, obj))
return False
return True
def _put_recursive(
container,
obj,
stream,
offset,
size,
):
stream.seek(offset)
headers = {'Content-Length': size}
try:
(status, reason, headers, contents) = \
self.client.put_object(container, obj, stream,
headers=headers)
if status != 201:
return False
except:
return False
return True
def setUp(self):
if Application is None:
raise SkipTest
self.client = Client(retries=1,
swift_proxy=True,
swift_proxy_storage_path=SWIFT_PROXY_STORAGE_PATH)
def setUp(self):
self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY, retries=1)
class TestObjectClient(unittest.TestCase):
def setUp(self):
self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY)
self.uuid = uuid4().hex
self.container = 'swiftly_test_' + self.uuid
status = self.client.put_container(self.container)[0]
self.assertTrue(status // 100 == 2)
status = \
self.client.put_object(self.container, 'object1', '123456789')[0]
self.assertTrue(status // 100 == 2)
def tearDown(self):
self.client.delete_object(self.container, 'object1')
self.client.delete_container(self.container)
def test_head(self):
status, reason, headers, contents = \
self.client.head_object(self.container, 'object1')
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertEquals(headers['etag'], '25f9e794323b453885f5181f1b624d0b')
self.assertTrue(isinstance(contents, str))
def test_get(self):
status, reason, headers, contents = \
self.client.get_object(self.container, 'object1')
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertEquals(headers['etag'], '25f9e794323b453885f5181f1b624d0b')
self.assertTrue(contents.read(), '123456789')
def test_get_no_stream(self):
status, reason, headers, contents = \
self.client.get_object(self.container, 'object1', stream=False)
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertEquals(headers['etag'], '25f9e794323b453885f5181f1b624d0b')
self.assertTrue(contents, '123456789')
def test_put(self):
try:
status, reason, headers, contents = self.client.put_object(
self.container,
'object2',
'contentvalue',
headers={'x-object-meta-swiftly-test': 'somevalue'})
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status, reason, headers, contents = \
self.client.get_object(self.container, 'object2')
self.assertTrue(status // 100 == 2)
self.assertEquals(headers['etag'],
'11a49846cb4877d7f9bb9471b646464f')
self.assertEquals(headers['x-object-meta-swiftly-test'],
'somevalue')
self.assertEquals(contents.read(), 'contentvalue')
finally:
self.client.delete_object(self.container, 'object2')
def test_put_with_etag(self):
try:
status, reason, headers, contents = self.client.put_object(
self.container,
'object2',
'contentvalue',
headers={
'x-object-meta-swiftly-test': 'somevalue',
'etag': '11111111111111111111111111111111'
})
self.assertEquals(status, 422)
status, reason, headers, contents = self.client.put_object(
self.container,
'object2',
'contentvalue',
headers={
'x-object-meta-swiftly-test': 'somevalue',
'etag': '11a49846cb4877d7f9bb9471b646464f'
})
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status, reason, headers, contents = \
self.client.get_object(self.container, 'object2')
self.assertTrue(status // 100 == 2)
self.assertEquals(headers['etag'],
'11a49846cb4877d7f9bb9471b646464f')
self.assertEquals(headers['x-object-meta-swiftly-test'],
'somevalue')
self.assertEquals(contents.read(), 'contentvalue')
finally:
self.client.delete_object(self.container, 'object2')
def test_put_as_stream(self):
try:
status, reason, headers, contents = self.client.put_object(
self.container,
'object2',
StringIO('contentvalue'),
headers={
'content-length': '12',
'x-object-meta-swiftly-test': 'somevalue'
})
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status, reason, headers, contents = \
self.client.get_object(self.container, 'object2')
self.assertTrue(status // 100 == 2)
self.assertEquals(headers['etag'],
'11a49846cb4877d7f9bb9471b646464f')
self.assertEquals(headers['x-object-meta-swiftly-test'],
'somevalue')
self.assertEquals(contents.read(), 'contentvalue')
finally:
self.client.delete_object(self.container, 'object2')
def test_put_as_stream_chunked(self):
try:
status, reason, headers, contents = self.client.put_object(
self.container,
'object2',
StringIO('contentvalue'),
headers={'x-object-meta-swiftly-test': 'somevalue'})
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status, reason, headers, contents = \
self.client.get_object(self.container, 'object2')
self.assertTrue(status // 100 == 2)
self.assertEquals(headers['etag'],
'11a49846cb4877d7f9bb9471b646464f')
self.assertEquals(headers['x-object-meta-swiftly-test'],
'somevalue')
self.assertEquals(contents.read(), 'contentvalue')
finally:
self.client.delete_object(self.container, 'object2')
def test_post(self):
status, reason, headers, contents = \
self.client.head_object(self.container, 'object1')
self.assertTrue(status // 100 == 2)
self.assertTrue('x-object-meta-swiftly-test' not in headers)
status, reason, headers, contents = self.client.post_object(
self.container,
'object1',
headers={'x-object-meta-swiftly-test': 'testvalue'})
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status, reason, headers, contents = \
self.client.head_object(self.container, 'object1')
self.assertTrue(status // 100 == 2)
self.assertEquals(headers['x-object-meta-swiftly-test'], 'testvalue')
def test_delete(self):
status = self.client.head_object(self.container, 'object1')[0]
self.assertTrue(status // 100 == 2)
status, reason, headers, contents = \
self.client.delete_object(self.container, 'object1')
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status = self.client.head_object(self.container, 'object1')[0]
self.assertEquals(status, 404)
class TestObjectClient(unittest.TestCase):
def setUp(self):
self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY)
self.uuid = uuid4().hex
self.container = 'swiftly_test_' + self.uuid
status = self.client.put_container(self.container)[0]
self.assertTrue(status // 100 == 2)
status = \
self.client.put_object(self.container, 'object1', '123456789')[0]
self.assertTrue(status // 100 == 2)
def tearDown(self):
self.client.delete_object(self.container, 'object1')
self.client.delete_container(self.container)
def test_head(self):
status, reason, headers, contents = \
self.client.head_object(self.container, 'object1')
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertEquals(headers['etag'], '25f9e794323b453885f5181f1b624d0b')
self.assertTrue(isinstance(contents, str))
def test_get(self):
status, reason, headers, contents = \
self.client.get_object(self.container, 'object1')
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertEquals(headers['etag'], '25f9e794323b453885f5181f1b624d0b')
self.assertTrue(contents.read(), '123456789')
def test_get_no_stream(self):
status, reason, headers, contents = \
self.client.get_object(self.container, 'object1', stream=False)
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertEquals(headers['etag'], '25f9e794323b453885f5181f1b624d0b')
self.assertTrue(contents, '123456789')
def test_put(self):
try:
status, reason, headers, contents = self.client.put_object(
self.container, 'object2', 'contentvalue',
headers={'x-object-meta-swiftly-test': 'somevalue'})
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status, reason, headers, contents = \
self.client.get_object(self.container, 'object2')
self.assertTrue(status // 100 == 2)
self.assertEquals(
headers['etag'], '11a49846cb4877d7f9bb9471b646464f')
self.assertEquals(
headers['x-object-meta-swiftly-test'], 'somevalue')
self.assertEquals(contents.read(), 'contentvalue')
finally:
self.client.delete_object(self.container, 'object2')
def test_put_with_etag(self):
try:
status, reason, headers, contents = self.client.put_object(
self.container, 'object2', 'contentvalue',
headers={'x-object-meta-swiftly-test': 'somevalue',
'etag': '11111111111111111111111111111111'})
self.assertEquals(status, 422)
status, reason, headers, contents = self.client.put_object(
self.container, 'object2', 'contentvalue',
headers={'x-object-meta-swiftly-test': 'somevalue',
'etag': '11a49846cb4877d7f9bb9471b646464f'})
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status, reason, headers, contents = \
self.client.get_object(self.container, 'object2')
self.assertTrue(status // 100 == 2)
self.assertEquals(
headers['etag'], '11a49846cb4877d7f9bb9471b646464f')
self.assertEquals(
headers['x-object-meta-swiftly-test'], 'somevalue')
self.assertEquals(contents.read(), 'contentvalue')
finally:
self.client.delete_object(self.container, 'object2')
def test_put_as_stream(self):
try:
status, reason, headers, contents = self.client.put_object(
self.container, 'object2', StringIO('contentvalue'),
headers={'content-length': '12',
'x-object-meta-swiftly-test': 'somevalue'})
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status, reason, headers, contents = \
self.client.get_object(self.container, 'object2')
self.assertTrue(status // 100 == 2)
self.assertEquals(
headers['etag'], '11a49846cb4877d7f9bb9471b646464f')
self.assertEquals(
headers['x-object-meta-swiftly-test'], 'somevalue')
self.assertEquals(contents.read(), 'contentvalue')
finally:
self.client.delete_object(self.container, 'object2')
def test_put_as_stream_chunked(self):
try:
status, reason, headers, contents = self.client.put_object(
self.container, 'object2', StringIO('contentvalue'),
headers={'x-object-meta-swiftly-test': 'somevalue'})
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status, reason, headers, contents = \
self.client.get_object(self.container, 'object2')
self.assertTrue(status // 100 == 2)
self.assertEquals(
headers['etag'], '11a49846cb4877d7f9bb9471b646464f')
self.assertEquals(
headers['x-object-meta-swiftly-test'], 'somevalue')
self.assertEquals(contents.read(), 'contentvalue')
finally:
self.client.delete_object(self.container, 'object2')
def test_post(self):
status, reason, headers, contents = \
self.client.head_object(self.container, 'object1')
self.assertTrue(status // 100 == 2)
self.assertTrue('x-object-meta-swiftly-test' not in headers)
status, reason, headers, contents = self.client.post_object(
self.container, 'object1',
headers={'x-object-meta-swiftly-test': 'testvalue'})
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status, reason, headers, contents = \
self.client.head_object(self.container, 'object1')
self.assertTrue(status // 100 == 2)
self.assertEquals(headers['x-object-meta-swiftly-test'], 'testvalue')
def test_delete(self):
status = self.client.head_object(self.container, 'object1')[0]
self.assertTrue(status // 100 == 2)
status, reason, headers, contents = \
self.client.delete_object(self.container, 'object1')
self.assertTrue(status // 100 == 2)
self.assertTrue(isinstance(reason, str))
self.assertTrue(isinstance(headers, dict))
self.assertTrue(isinstance(contents, str))
status = self.client.head_object(self.container, 'object1')[0]
self.assertEquals(status, 404)
def setUp(self):
self.client = Client(AUTH_URL, AUTH_USER, AUTH_KEY, retries=1)
class SwiftTransport(object):
name = 'swift'
def __init__(self, conf):
self.conf = conf
self.logger = get_logger(conf, log_route='transport.memcached')
self.auth_url = conf.get('auth_url', 'http://localhost:5000/')
self.auth_user = conf.get('auth_user', 'gate')
self.auth_key = conf.get('auth_key', 'gate')
self.proxy = conf.get('proxy', 'None')
if self.proxy == 'None':
self.proxy = None
self.retries = int(conf.get('retries', '4'))
self.swift_proxy = conf.get('swift_proxy', 'None')
if self.swift_proxy == 'None':
self.swift_proxy = None
self.cache_path = conf.get('cache_path',
'/var/gate/object-store/cache')
dir_path = os.path.dirname(os.path.abspath(cache_path))
if not os.path.exists(dir_path):
os.makedirs(dir_path)
self.concurrency = int(conf.get('put_threads', 5))
self.segment_size = int(conf.get('segment_size', 262144000))
def open(self):
self.client = Client(
auth_url,
auth_user,
auth_key,
proxy=proxy,
retries=retries,
swift_proxy=swift_proxy,
cache_path=cache_path,
)
try:
(status, reason, headers, contents) = \
__swiftly_client.head_account()
except:
self.logger.error('Connection failed: connection refused.')
return False
if status != 200:
self.logger.error('Connection failed: (%d) %s' % (status,
reason))
return False
self.logger.debug('Connection made to swift.')
return True
def close(self):
self.client.close()
self.logger.debug('Connection closed.')
def get_stream(self, key):
(container, obj) = key.split('/', limit=1)
return self.client.get_object(container, obj, stream=True)
def put_stream(self, key, stream):
(container, obj) = key.split('/', limit=1)
size = None
func_size = getattr(stream, 'size', None)
if func_size:
size = func_size()
if not size:
return self.client.client.put_object(container, obj, stream)
if size <= self.segment_size:
headers = {'Content-Length': size}
(status, reason, headers, contents) = \
self.client.put_object(container, obj, stream,
headers=headers)
if status != 201:
self.logger.error('Failed to put object: %s/%s'
% (container, obj))
return False
cont_prefix = '%s_segments' % container
prefix = '%s/%s/' % (obj, self.segment_size)
conc = Concurrency(self.concurrency)
start = 0
segment = 0
while start < size:
obj_path = '%s%08d' % (prefix, segment)
conc.spawn(
segment,
self._put_recursive,
cont_prefix,
obj_path,
stream.copy(),
start,
)
for rv in conc.get_results().values():
if not rv:
conc.join()
self.logger.error('Failed to create segments for object: %s/%s'
% (container, obj))
return rv
segment += 1
start += self.client.segment_size
conc.join()
for rv in conc.get_results().values():
if not rv:
conc.join()
self.logger.error('Failed to create segments for object: %s/%s'
% (container, obj))
return rv
headers = {'Content-Length': 0, 'X-Object-Manifest': '%s/%s' \
% (cont_prefix, prefix)}
(status, reason, headers, contents) = \
self.client.put_object(container, obj, '', headers=headers)
if status != 201:
self.logger.error('Failed put manifest object: %s/%s'
% (container, obj))
return False
return True
def _put_recursive(
container,
obj,
stream,
offset,
size,
):
stream.seek(offset)
headers = {'Content-Length': size}
try:
(status, reason, headers, contents) = \
self.client.put_object(container, obj, stream,
headers=headers)
if status != 201:
return False
except:
return False
return True
