def test_http_store_location_initialization(self):
"""Test store location initialization from valid uris"""
uris = [
"http://127.0.0.1:8000/ubuntu.iso",
"http://openstack.com:80/ubuntu.iso",
"http://[1080::8:800:200C:417A]:80/ubuntu.iso"
]
for uri in uris:
location.get_location_from_uri(uri)
def test_http_store_location_initialization(self):
"""Test store location initialization from valid uris"""
uris = [
"http://127.0.0.1:8000/ubuntu.iso",
"http://openstack.com:80/ubuntu.iso",
"http://[1080::8:800:200C:417A]:80/ubuntu.iso"
]
for uri in uris:
location.get_location_from_uri(uri)
def test_get_non_existing(self):
"""
Test that trying to retrieve a s3 that doesn't exist
raises an error
"""
uri = "s3://user:key@auth_address/badbucket/%s" % FAKE_UUID
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
uri = "s3://user:key@auth_address/glance/noexist"
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
def test_get_non_existing(self):
"""
Test that trying to retrieve a s3 that doesn't exist
raises an error
"""
uri = "s3://user:key@auth_address/badbucket/%s" % FAKE_UUID
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
uri = "s3://user:key@auth_address/glance/noexist"
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
def test_cinder_get_size(self):
fake_client = FakeObject(auth_token=None, management_url=None)
fake_volumes = {
'12345678-9012-3455-6789-012345678901': FakeObject(size=5)
}
with mock.patch.object(cinder, 'get_cinderclient') as mocked_cc:
mocked_cc.return_value = FakeObject(client=fake_client,
volumes=fake_volumes)
fake_sc = [{
u'endpoints': [{
u'publicURL': u'foo_public_url'
}],
u'endpoints_links': [],
u'name': u'cinder',
u'type': u'volume'
}]
fake_context = FakeObject(service_catalog=fake_sc,
user='******',
auth_tok='fake_token',
tenant='fake_tenant')
uri = 'cinder://%s' % fake_volumes.keys()[0]
loc = location.get_location_from_uri(uri, conf=self.conf)
image_size = self.store.get_size(loc, context=fake_context)
self.assertEqual(image_size,
fake_volumes.values()[0].size * (1024**3))
def test_add(self):
"""Test that we can add an image via the swift backend."""
reload(swift)
self.store = Store(self.conf)
self.store.configure()
expected_swift_size = FIVE_KB
expected_swift_contents = "*" * expected_swift_size
expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
expected_image_id = str(uuid.uuid4())
loc = "swift+https://tenant%%3Auser1:key@localhost:8080/glance/%s"
expected_location = loc % (expected_image_id)
image_swift = six.StringIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0
loc, size, checksum, _ = self.store.add(expected_image_id, image_swift,
expected_swift_size)
self.assertEqual(expected_location, loc)
self.assertEqual(expected_swift_size, size)
self.assertEqual(expected_checksum, checksum)
# Expecting a single object to be created on Swift i.e. no chunking.
self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1)
loc = location.get_location_from_uri(expected_location, conf=self.conf)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = ''.join([chunk for chunk in new_image_swift])
new_image_swift_size = len(new_image_swift)
self.assertEqual(expected_swift_contents, new_image_contents)
self.assertEqual(expected_swift_size, new_image_swift_size)
def test_http_get_size_bad_status_line(self):
self._mock_httplib()
self.response.side_effect = http_client.BadStatusLine(line='')
uri = "http://netloc/path/to/file.tar.gz"
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.BadStoreUri, self.store.get_size, loc)
def test_add(self):
"""Test that we can add an image via the filesystem backend."""
filesystem.ChunkedFile.CHUNKSIZE = units.Ki
expected_image_id = str(uuid.uuid4())
expected_file_size = 5 * units.Ki # 5K
expected_file_contents = b"*" * expected_file_size
expected_checksum = hashlib.md5(expected_file_contents).hexdigest()
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (self.test_dir, expected_image_id)
image_file = six.BytesIO(expected_file_contents)
loc, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo)
self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size)
self.assertEqual(expected_checksum, checksum)
self.assertEqual(expected_multihash, multihash)
uri = "file:///%s/%s" % (self.test_dir, expected_image_id)
loc = location.get_location_from_uri(uri, conf=self.conf)
(new_image_file, new_image_size) = self.store.get(loc)
new_image_contents = b""
new_image_file_size = 0
for chunk in new_image_file:
new_image_file_size += len(chunk)
new_image_contents += chunk
self.assertEqual(expected_file_contents, new_image_contents)
self.assertEqual(expected_file_size, new_image_file_size)
def test_http_get_redirect(self):
# Add two layers of redirects to the response stack, which will
# return the default 200 OK with the expected data after resolving
# both redirects.
self._mock_httplib()
redirect1 = {"location": "http://example.com/teapot.img"}
redirect2 = {"location": "http://example.com/teapot_real.img"}
responses = [
utils.FakeHTTPResponse(status=302, headers=redirect1),
utils.FakeHTTPResponse(status=301, headers=redirect2),
utils.FakeHTTPResponse()
]
def getresponse():
return responses.pop()
self.response.side_effect = getresponse
uri = "http://netloc/path/to/file.tar.gz"
expected_returns = [
'I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s', 'ho', 'rt', ' a',
'nd', ' s', 'to', 'ut', '\n'
]
loc = location.get_location_from_uri(uri, conf=self.conf)
(image_file, image_size) = self.store.get(loc)
self.assertEqual(image_size, 31)
chunks = [c for c in image_file]
self.assertEqual(chunks, expected_returns)
def test_add(self):
"""Test that we can add an image via the filesystem backend."""
ChunkedFile.CHUNKSIZE = units.Ki
expected_image_id = str(uuid.uuid4())
expected_file_size = 5 * units.Ki # 5K
expected_file_contents = "*" * expected_file_size
expected_checksum = hashlib.md5(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (self.test_dir,
expected_image_id)
image_file = StringIO.StringIO(expected_file_contents)
loc, size, checksum, _ = self.store.add(expected_image_id,
image_file,
expected_file_size)
self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size)
self.assertEqual(expected_checksum, checksum)
uri = "file:///%s/%s" % (self.test_dir, expected_image_id)
loc = location.get_location_from_uri(uri, conf=self.conf)
(new_image_file, new_image_size) = self.store.get(loc)
new_image_contents = ""
new_image_file_size = 0
for chunk in new_image_file:
new_image_file_size += len(chunk)
new_image_contents += chunk
self.assertEqual(expected_file_contents, new_image_contents)
self.assertEqual(expected_file_size, new_image_file_size)
def test_delete_forbidden(self):
"""
Tests that trying to delete a file without permissions
raises the correct error
"""
# First add an image
image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K
file_contents = "*" * file_size
image_file = StringIO.StringIO(file_contents)
loc, size, checksum, _ = self.store.add(image_id,
image_file,
file_size)
uri = "file:///%s/%s" % (self.test_dir, image_id)
loc = location.get_location_from_uri(uri, conf=self.conf)
# Mock unlink to raise an OSError for lack of permissions
# and make sure we can't delete the image
with mock.patch.object(os, 'unlink') as unlink:
e = OSError()
e.errno = errno
unlink.side_effect = e
self.assertRaises(exceptions.Forbidden,
self.store.delete,
loc)
# Make sure the image didn't get deleted
self.store.get(loc)
def test_partial_get(self):
loc = location.get_location_from_uri(
"s3://user:key@auth_address/glance/%s" % FAKE_UUID, conf=self.conf)
self.assertRaises(exceptions.StoreRandomGetNotSupported,
self.store.get,
loc,
chunk_size=1)
def test_get_random_access(self):
"""Test a "normal" retrieval of an image in chunks."""
# First add an image...
image_id = str(uuid.uuid4())
file_contents = "chunk00000remainder"
image_file = StringIO.StringIO(file_contents)
loc, size, checksum, _ = self.store.add(image_id,
image_file,
len(file_contents))
# Now read it back...
uri = "file:///%s/%s" % (self.test_dir, image_id)
loc = location.get_location_from_uri(uri, conf=self.conf)
data = ""
for offset in range(len(file_contents)):
(image_file, image_size) = self.store.get(loc,
offset=offset,
chunk_size=1)
for chunk in image_file:
data += chunk
self.assertEqual(data, file_contents)
data = ""
chunk_size = 5
(image_file, image_size) = self.store.get(loc,
offset=chunk_size,
chunk_size=chunk_size)
for chunk in image_file:
data += chunk
self.assertEqual(data, '00000')
self.assertEqual(image_size, chunk_size)
def get_size_from_backend(uri, context=None):
"""Retrieves image size from backend specified by uri"""
loc = location.get_location_from_uri(uri)
store = get_store_from_uri(uri)
return store.get_size(loc, context=context)
def test_image_get_with_proxy(self):
"""Test s3 get with proxy connection."""
self.config(s3_store_enable_proxy=True)
proxy_host = '127.0.0.1'
self.config(s3_store_proxy_host=proxy_host)
with mock.patch.object(boto.s3.connection, 'S3Connection') as m:
cf = s3.get_calling_format(bucket_format=None,
s3_store_bucket_url_format='subdomain')
with mock.patch.object(s3, 'get_calling_format') as gcf:
gcf.return_value = cf
loc = location.get_location_from_uri(
"s3://user:key@auth_address/glance/%s" % FAKE_UUID,
conf=self.conf)
self.store.get(loc)
accesskey = S3_CONF['s3_store_access_key']
secretkey = S3_CONF['s3_store_secret_key']
proxy_port = S3_CONF['s3_store_proxy_port']
proxy_pass = S3_CONF['s3_store_proxy_password']
proxy_user = S3_CONF['s3_store_proxy_user']
m.assert_called_with(accesskey, secretkey,
calling_format=cf,
is_secure=False,
proxy=proxy_host,
proxy_pass=proxy_pass,
proxy_port=proxy_port,
proxy_user=proxy_user)
def test_http_partial_get(self):
uri = "http://netloc/path/to/file.tar.gz"
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.StoreRandomGetNotSupported,
self.store.get,
loc,
chunk_size=1)
def test_get(self):
"""Test a "normal" retrieval of an image in chunks."""
# First add an image...
image_id = str(uuid.uuid4())
file_contents = "chunk00000remainder"
image_file = StringIO.StringIO(file_contents)
loc, size, checksum, _ = self.store.add(image_id,
image_file,
len(file_contents))
# Now read it back...
uri = "file:///%s/%s" % (self.test_dir, image_id)
loc = location.get_location_from_uri(uri, conf=self.conf)
(image_file, image_size) = self.store.get(loc)
expected_data = "chunk00000remainder"
expected_num_chunks = 2
data = ""
num_chunks = 0
for chunk in image_file:
num_chunks += 1
data += chunk
self.assertEqual(expected_data, data)
self.assertEqual(expected_num_chunks, num_chunks)
def test_get_random_access(self):
"""Test a "normal" retrieval of an image in chunks."""
# First add an image...
image_id = str(uuid.uuid4())
file_contents = b"chunk00000remainder"
image_file = six.BytesIO(file_contents)
loc, size, checksum, multihash, _ = self.store.add(
image_id, image_file, len(file_contents), self.hash_algo)
# Now read it back...
uri = "file:///%s/%s" % (self.test_dir, image_id)
loc = location.get_location_from_uri(uri, conf=self.conf)
data = b""
for offset in range(len(file_contents)):
(image_file, image_size) = self.store.get(loc,
offset=offset,
chunk_size=1)
for chunk in image_file:
data += chunk
self.assertEqual(file_contents, data)
data = b""
chunk_size = 5
(image_file, image_size) = self.store.get(loc,
offset=chunk_size,
chunk_size=chunk_size)
for chunk in image_file:
data += chunk
self.assertEqual(b'00000', data)
self.assertEqual(chunk_size, image_size)
def test_add(self):
"""Test that we can add an image via the filesystem backend"""
ChunkedFile.CHUNKSIZE = 1024
expected_image_id = str(uuid.uuid4())
expected_file_size = 5 * KB # 5K
expected_file_contents = "*" * expected_file_size
expected_checksum = hashlib.md5(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (self.test_dir, expected_image_id)
image_file = StringIO.StringIO(expected_file_contents)
location, size, checksum, _ = self.store.add(expected_image_id,
image_file,
expected_file_size)
self.assertEqual(expected_location, location)
self.assertEqual(expected_file_size, size)
self.assertEqual(expected_checksum, checksum)
uri = "file:///%s/%s" % (self.test_dir, expected_image_id)
loc = get_location_from_uri(uri)
(new_image_file, new_image_size) = self.store.get(loc)
new_image_contents = ""
new_image_file_size = 0
for chunk in new_image_file:
new_image_file_size += len(chunk)
new_image_contents += chunk
self.assertEqual(expected_file_contents, new_image_contents)
self.assertEqual(expected_file_size, new_image_file_size)
def get_size_from_backend(uri, context=None):
"""Retrieves image size from backend specified by uri"""
loc = location.get_location_from_uri(uri)
store = get_store_from_uri(uri)
return store.get_size(loc, context=context)
def test_http_get(self):
uri = "http://netloc/path/to/file.tar.gz"
expected_returns = [
"I ",
"am",
" a",
" t",
"ea",
"po",
"t,",
" s",
"ho",
"rt",
" a",
"nd",
" s",
"to",
"ut",
"\n",
]
loc = get_location_from_uri(uri)
(image_file, image_size) = self.store.get(loc)
self.assertEqual(image_size, 31)
chunks = [c for c in image_file]
self.assertEqual(chunks, expected_returns)
def test_partial_get(self):
"""Test a "normal" retrieval of an image in chunks."""
loc = location.get_location_from_uri(
"s3://user:key@auth_address/glance/%s" % FAKE_UUID,
conf=self.conf)
self.assertRaises(exceptions.StoreRandomGetNotSupported,
self.store.get, loc, chunk_size=1)
def test_http_get_not_found(self):
fake = utils.FakeHTTPResponse(status=404, data="404 Not Found")
self.response.return_value = fake
uri = "http://netloc/path/to/file.tar.gz"
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
def test_http_get_redirect(self):
# Add two layers of redirects to the response stack, which will
# return the default 200 OK with the expected data after resolving
# both redirects.
self._mock_httplib()
redirect1 = {"location": "http://example.com/teapot.img"}
redirect2 = {"location": "http://example.com/teapot_real.img"}
responses = [utils.FakeHTTPResponse(status=302, headers=redirect1),
utils.FakeHTTPResponse(status=301, headers=redirect2),
utils.FakeHTTPResponse()]
def getresponse():
return responses.pop()
self.response.side_effect = getresponse
uri = "http://netloc/path/to/file.tar.gz"
expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s',
'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n']
loc = location.get_location_from_uri(uri, conf=self.conf)
(image_file, image_size) = self.store.get(loc)
self.assertEqual(image_size, 31)
chunks = [c for c in image_file]
self.assertEqual(chunks, expected_returns)
def test_add(self):
"""Test that we can add an image via the swift backend."""
reload(swift)
self.store = Store(self.conf)
self.store.configure()
expected_swift_size = FIVE_KB
expected_swift_contents = "*" * expected_swift_size
expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
expected_image_id = str(uuid.uuid4())
loc = "swift+https://tenant%%3Auser1:key@localhost:8080/glance/%s"
expected_location = loc % (expected_image_id)
image_swift = six.StringIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0
loc, size, checksum, _ = self.store.add(expected_image_id,
image_swift,
expected_swift_size)
self.assertEqual(expected_location, loc)
self.assertEqual(expected_swift_size, size)
self.assertEqual(expected_checksum, checksum)
# Expecting a single object to be created on Swift i.e. no chunking.
self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1)
loc = location.get_location_from_uri(expected_location, conf=self.conf)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = ''.join([chunk for chunk in new_image_swift])
new_image_swift_size = len(new_image_swift)
self.assertEqual(expected_swift_contents, new_image_contents)
self.assertEqual(expected_swift_size, new_image_swift_size)
def test_cinder_delete_raise_error(self):
uri = 'cinder://12345678-9012-3455-6789-012345678901'
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.StoreDeleteNotSupported,
self.store.delete, loc)
self.assertRaises(exceptions.StoreDeleteNotSupported,
glance_store.delete_from_backend, uri, {})
def test_add(self):
"""Test that we can add an image via the s3 backend."""
expected_image_id = str(uuid.uuid4())
expected_s3_size = FIVE_KB
expected_s3_contents = b"*" * expected_s3_size
expected_checksum = hashlib.md5(expected_s3_contents).hexdigest()
expected_location = format_s3_location(S3_CONF['s3_store_access_key'],
S3_CONF['s3_store_secret_key'],
S3_CONF['s3_store_host'],
S3_CONF['s3_store_bucket'],
expected_image_id)
image_s3 = six.BytesIO(expected_s3_contents)
loc, size, checksum, _ = self.store.add(expected_image_id, image_s3,
expected_s3_size)
self.assertEqual(expected_location, loc)
self.assertEqual(expected_s3_size, size)
self.assertEqual(expected_checksum, checksum)
loc = location.get_location_from_uri(expected_location, conf=self.conf)
(new_image_s3, new_image_size) = self.store.get(loc)
new_image_contents = six.BytesIO()
for chunk in new_image_s3:
new_image_contents.write(chunk)
new_image_s3_size = new_image_contents.tell()
self.assertEqual(expected_s3_contents, new_image_contents.getvalue())
self.assertEqual(expected_s3_size, new_image_s3_size)
def test_cinder_delete_raise_error(self):
uri = 'cinder://12345678-9012-3455-6789-012345678901'
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.StoreDeleteNotSupported,
self.store.delete, loc)
self.assertRaises(exceptions.StoreDeleteNotSupported,
glance_store.delete_from_backend, uri, {})
def test_http_get_not_found(self):
fake = utils.FakeHTTPResponse(status=404, data="404 Not Found")
self.response.return_value = fake
uri = "http://netloc/path/to/file.tar.gz"
loc = get_location_from_uri(uri)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
def test_delete_forbidden(self):
"""
Tests that trying to delete a file without permissions
raises the correct error
"""
# First add an image
image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents)
loc, size, checksum, multihash, _ = self.store.add(
image_id, image_file, file_size, self.hash_algo)
uri = "file:///%s/%s" % (self.test_dir, image_id)
loc = location.get_location_from_uri(uri, conf=self.conf)
# Mock unlink to raise an OSError for lack of permissions
# and make sure we can't delete the image
with mock.patch.object(os, 'unlink') as unlink:
e = OSError()
e.errno = errno
unlink.side_effect = e
self.assertRaises(exceptions.Forbidden, self.store.delete, loc)
# Make sure the image didn't get deleted
self.store.get(loc)
def test_get_random_access(self):
"""Test a "normal" retrieval of an image in chunks."""
# First add an image...
image_id = str(uuid.uuid4())
file_contents = "chunk00000remainder"
image_file = StringIO.StringIO(file_contents)
location, size, checksum, _ = self.store.add(image_id, image_file,
len(file_contents))
# Now read it back...
uri = "file:///%s/%s" % (self.test_dir, image_id)
loc = get_location_from_uri(uri)
data = ""
for offset in range(len(file_contents)):
(image_file, image_size) = self.store.get(loc,
offset=offset,
chunk_size=1)
for chunk in image_file:
data += chunk
self.assertEqual(data, file_contents)
data = ""
chunk_size = 5
(image_file, image_size) = self.store.get(loc,
offset=chunk_size,
chunk_size=chunk_size)
for chunk in image_file:
data += chunk
self.assertEqual(data, '00000')
self.assertEqual(image_size, chunk_size)
def test_delete_non_existing(self):
"""
Test that trying to delete a file that doesn't exist
raises an error
"""
loc = get_location_from_uri("file:///tmp/glance-tests/non-existing")
self.assertRaises(exceptions.NotFound, self.store.delete, loc)
def test_http_get_redirect(self, mock_api_session):
# Add two layers of redirects to the response stack, which will
# return the default 200 OK with the expected data after resolving
# both redirects.
redirect1 = {"location": "https://example.com?dsName=ds1&dcPath=dc1"}
redirect2 = {"location": "https://example.com?dsName=ds2&dcPath=dc2"}
responses = [
utils.fake_response(),
utils.fake_response(status_code=302, headers=redirect1),
utils.fake_response(status_code=301, headers=redirect2)
]
def getresponse(*args, **kwargs):
return responses.pop()
expected_image_size = 31
expected_returns = ['I am a teapot, short and stout\n']
loc = location.get_location_from_uri(
"vsphere://127.0.0.1/folder/openstack_glance/%s"
"?dsName=ds1&dcPath=dc1" % FAKE_UUID,
conf=self.conf)
with mock.patch('requests.Session.request') as HttpConn:
HttpConn.side_effect = getresponse
(image_file, image_size) = self.store.get(loc)
self.assertEqual(image_size, expected_image_size)
chunks = [c for c in image_file]
self.assertEqual(expected_returns, chunks)
def test_image_get_with_proxy(self):
"""Test s3 get with proxy connection."""
self.config(s3_store_enable_proxy=True)
proxy_host = '127.0.0.1'
self.config(s3_store_proxy_host=proxy_host)
with mock.patch.object(boto.s3.connection, 'S3Connection') as m:
cf = s3.get_calling_format(bucket_format=None,
s3_store_bucket_url_format='subdomain')
with mock.patch.object(s3, 'get_calling_format') as gcf:
gcf.return_value = cf
loc = location.get_location_from_uri(
"s3://user:key@auth_address/glance/%s" % FAKE_UUID,
conf=self.conf)
self.store.get(loc)
accesskey = S3_CONF['s3_store_access_key']
secretkey = S3_CONF['s3_store_secret_key']
proxy_port = S3_CONF['s3_store_proxy_port']
proxy_pass = S3_CONF['s3_store_proxy_password']
proxy_user = S3_CONF['s3_store_proxy_user']
m.assert_called_with(accesskey,
secretkey,
calling_format=cf,
is_secure=False,
proxy=proxy_host,
proxy_pass=proxy_pass,
proxy_port=proxy_port,
proxy_user=proxy_user)
def test_get(self):
"""Test a "normal" retrieval of an image in chunks."""
# First add an image...
image_id = str(uuid.uuid4())
file_contents = "chunk00000remainder"
image_file = StringIO.StringIO(file_contents)
location, size, checksum, _ = self.store.add(image_id, image_file,
len(file_contents))
# Now read it back...
uri = "file:///%s/%s" % (self.test_dir, image_id)
loc = get_location_from_uri(uri)
(image_file, image_size) = self.store.get(loc)
expected_data = "chunk00000remainder"
expected_num_chunks = 2
data = ""
num_chunks = 0
for chunk in image_file:
num_chunks += 1
data += chunk
self.assertEqual(expected_data, data)
self.assertEqual(expected_num_chunks, num_chunks)
def test_get_non_existing(self):
"""
Test that trying to retrieve a file that doesn't exist
raises an error
"""
loc = get_location_from_uri("file:///%s/non-existing" % self.test_dir)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
def test_http_delete_raise_error(self):
self._mock_httplib()
uri = "https://netloc/path/to/file.tar.gz"
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(NotImplementedError, self.store.delete, loc)
self.assertRaises(exceptions.StoreDeleteNotSupported,
delete_from_backend, uri, {})
def test_add(self):
"""Test that we can add an image via the s3 backend."""
expected_image_id = str(uuid.uuid4())
expected_s3_size = FIVE_KB
expected_s3_contents = "*" * expected_s3_size
expected_checksum = hashlib.md5(expected_s3_contents).hexdigest()
expected_location = format_s3_location(
S3_CONF['s3_store_access_key'],
S3_CONF['s3_store_secret_key'],
S3_CONF['s3_store_host'],
S3_CONF['s3_store_bucket'],
expected_image_id)
image_s3 = six.StringIO(expected_s3_contents)
loc, size, checksum, _ = self.store.add(expected_image_id,
image_s3,
expected_s3_size)
self.assertEqual(expected_location, loc)
self.assertEqual(expected_s3_size, size)
self.assertEqual(expected_checksum, checksum)
loc = location.get_location_from_uri(expected_location,
conf=self.conf)
(new_image_s3, new_image_size) = self.store.get(loc)
new_image_contents = six.StringIO()
for chunk in new_image_s3:
new_image_contents.write(chunk)
new_image_s3_size = new_image_contents.len
self.assertEqual(expected_s3_contents, new_image_contents.getvalue())
self.assertEqual(expected_s3_size, new_image_s3_size)
def test_add_large_object_zero_size(self):
"""
Tests that adding an image to Swift which has both an unknown size and
exceeds Swift's maximum limit of 5GB is correctly uploaded.
We avoid the overhead of creating a 5GB object for this test by
temporarily setting MAX_SWIFT_OBJECT_SIZE to 1KB, and then adding
an object of 5KB.
Bug lp:891738
"""
# Set up a 'large' image of 5KB
expected_swift_size = FIVE_KB
expected_swift_contents = "*" * expected_swift_size
expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
expected_image_id = str(uuid.uuid4())
loc = 'swift+config://ref1/glance/%s'
expected_location = loc % (expected_image_id)
image_swift = six.StringIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0
# Temporarily set Swift MAX_SWIFT_OBJECT_SIZE to 1KB and add our image,
# explicitly setting the image_length to 0
self.store = Store(self.conf)
self.store.configure()
orig_max_size = self.store.large_object_size
orig_temp_size = self.store.large_object_chunk_size
global MAX_SWIFT_OBJECT_SIZE
orig_max_swift_object_size = MAX_SWIFT_OBJECT_SIZE
try:
MAX_SWIFT_OBJECT_SIZE = 1024
self.store.large_object_size = 1024
self.store.large_object_chunk_size = 1024
loc, size, checksum, _ = self.store.add(expected_image_id,
image_swift, 0)
finally:
self.store.large_object_chunk_size = orig_temp_size
self.store.large_object_size = orig_max_size
MAX_SWIFT_OBJECT_SIZE = orig_max_swift_object_size
self.assertEqual(expected_location, loc)
self.assertEqual(expected_swift_size, size)
self.assertEqual(expected_checksum, checksum)
# Expecting 7 calls to put_object -- 5 chunks, a zero chunk which is
# then deleted, and the manifest. Note the difference with above
# where the image_size is specified in advance (there's no zero chunk
# in that case).
self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 7)
loc = location.get_location_from_uri(expected_location, conf=self.conf)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = ''.join([chunk for chunk in new_image_swift])
new_image_swift_size = len(new_image_contents)
self.assertEqual(expected_swift_contents, new_image_contents)
self.assertEqual(expected_swift_size, new_image_swift_size)
def test_add_large_object_zero_size(self):
"""
Tests that adding an image to Swift which has both an unknown size and
exceeds Swift's maximum limit of 5GB is correctly uploaded.
We avoid the overhead of creating a 5GB object for this test by
temporarily setting MAX_SWIFT_OBJECT_SIZE to 1KB, and then adding
an object of 5KB.
Bug lp:891738
"""
# Set up a 'large' image of 5KB
expected_swift_size = FIVE_KB
expected_swift_contents = "*" * expected_swift_size
expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
expected_image_id = str(uuid.uuid4())
loc = 'swift+config://ref1/glance/%s'
expected_location = loc % (expected_image_id)
image_swift = six.StringIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0
# Temporarily set Swift MAX_SWIFT_OBJECT_SIZE to 1KB and add our image,
# explicitly setting the image_length to 0
self.store = Store(self.conf)
self.store.configure()
orig_max_size = self.store.large_object_size
orig_temp_size = self.store.large_object_chunk_size
global MAX_SWIFT_OBJECT_SIZE
orig_max_swift_object_size = MAX_SWIFT_OBJECT_SIZE
try:
MAX_SWIFT_OBJECT_SIZE = 1024
self.store.large_object_size = 1024
self.store.large_object_chunk_size = 1024
loc, size, checksum, _ = self.store.add(expected_image_id,
image_swift, 0)
finally:
self.store.large_object_chunk_size = orig_temp_size
self.store.large_object_size = orig_max_size
MAX_SWIFT_OBJECT_SIZE = orig_max_swift_object_size
self.assertEqual(expected_location, loc)
self.assertEqual(expected_swift_size, size)
self.assertEqual(expected_checksum, checksum)
# Expecting 7 calls to put_object -- 5 chunks, a zero chunk which is
# then deleted, and the manifest. Note the difference with above
# where the image_size is specified in advance (there's no zero chunk
# in that case).
self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 7)
loc = location.get_location_from_uri(expected_location, conf=self.conf)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = ''.join([chunk for chunk in new_image_swift])
new_image_swift_size = len(new_image_contents)
self.assertEqual(expected_swift_contents, new_image_contents)
self.assertEqual(expected_swift_size, new_image_swift_size)
def test_http_get_redirect_invalid(self):
redirect = {"location": "http://example.com/teapot.img"}
redirect_resp = utils.FakeHTTPResponse(status=307, headers=redirect)
self.response.return_value = redirect_resp
uri = "http://netloc/path/to/file.tar.gz"
loc = get_location_from_uri(uri)
self.assertRaises(exceptions.BadStoreUri, self.store.get, loc)
def test_delete_non_existing(self):
"""
Test that trying to delete a s3 that doesn't exist
raises an error
"""
uri = "s3://user:key@auth_address/glance/noexist"
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.delete, loc)
def test_http_get_not_found(self):
self._mock_requests()
fake = utils.fake_response(status_code=404, content="404 Not Found")
self.request.return_value = fake
uri = "http://netloc/path/to/file.tar.gz"
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
def test_image_get_with_proxy_without_host(self):
"""Test s3 backend with unconfigured proxy connection."""
self.config(s3_store_enable_proxy=True)
loc = location.get_location_from_uri(
"s3://user:key@auth_address/glance/%s" % FAKE_UUID, conf=self.conf)
self.assertRaises(exceptions.BadStoreConfiguration, self.store.get,
loc)
def test_http_get_redirect_invalid(self):
redirect = {"location": "http://example.com/teapot.img"}
redirect_resp = utils.FakeHTTPResponse(status=307, headers=redirect)
self.response.return_value = redirect_resp
uri = "http://netloc/path/to/file.tar.gz"
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.BadStoreUri, self.store.get, loc)
def get_size_from_backend(uri, context=None):
"""Retrieves image size from backend specified by uri"""
if is_glance_location(uri):
uri += ('?auth_token=' + context.auth_tok)
loc = location.get_location_from_uri(uri)
store = get_store_from_uri(uri)
return store.get_size(loc, context=context)
def test_delete_non_existing(self):
"""
Test that trying to delete a s3 that doesn't exist
raises an error
"""
uri = "s3://user:key@auth_address/glance/noexist"
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.delete, loc)
def test_delete_non_existing(self):
"""
Test that trying to delete a swift that doesn't exist
raises an error
"""
loc = get_location_from_uri("swift://%s:key@authurl/glance/noexist" %
(self.swift_store_user))
self.assertRaises(exceptions.NotFound, self.store.delete, loc)
def test_http_get_size_with_non_existent_image_raises_Not_Found(self):
self._mock_httplib()
fake = utils.FakeHTTPResponse(status=404, data="404 Not Found")
self.response.return_value = fake
uri = "http://netloc/path/to/file.tar.gz"
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.get_size, loc)
def test_http_get_not_found(self):
self._mock_requests()
fake = utils.fake_response(status_code=404, content="404 Not Found")
self.request.return_value = fake
uri = "http://netloc/path/to/file.tar.gz"
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
def test_delete_non_existing(self):
"""
Test that trying to delete a swift that doesn't exist
raises an error
"""
loc = get_location_from_uri("swift://%s:key@authurl/glance/noexist" % (
self.swift_store_user))
self.assertRaises(exceptions.NotFound, self.store.delete, loc)
def test_http_get_size_with_non_existent_image_raises_Not_Found(self):
self._mock_httplib()
fake = utils.FakeHTTPResponse(status=404, data="404 Not Found")
self.response.return_value = fake
uri = "http://netloc/path/to/file.tar.gz"
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.get_size, loc)
def test_http_delete_raise_error(self):
self._mock_httplib()
uri = "https://netloc/path/to/file.tar.gz"
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.StoreDeleteNotSupported,
self.store.delete, loc)
self.assertRaises(exceptions.StoreDeleteNotSupported,
glance_store.delete_from_backend, uri, {})
def test_image_get_with_proxy_without_host(self):
"""Test s3 backend with unconfigured proxy connection."""
self.config(s3_store_enable_proxy=True)
loc = location.get_location_from_uri(
"s3://user:key@auth_address/glance/%s" % FAKE_UUID,
conf=self.conf)
self.assertRaises(exceptions.BadStoreConfiguration,
self.store.get, loc)
def delete_from_backend(uri, context=None):
"""Removes chunks of data from backend specified by uri"""
loc = location.get_location_from_uri(uri)
store = get_store_from_uri(uri)
try:
return store.delete(loc, context=context)
except NotImplementedError:
raise exceptions.StoreDeleteNotSupported
def test_delete(self):
"""
Test we can delete an existing image in the s3 store
"""
uri = "s3://user:key@auth_address/glance/%s" % FAKE_UUID
loc = location.get_location_from_uri(uri, conf=self.conf)
self.store.delete(loc)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
def test_get_size(self):
"""
Test that we can get the size of an object in the swift store
"""
uri = "swift://%s:key@auth_address/glance/%s" % (
self.swift_store_user, FAKE_UUID)
loc = location.get_location_from_uri(uri, conf=self.conf)
image_size = self.store.get_size(loc)
self.assertEqual(image_size, 5120)
def test_delete_with_reference_params(self):
"""
Test we can delete an existing image in the swift store
"""
uri = "swift+config://ref1/glance/%s" % (FAKE_UUID)
loc = location.get_location_from_uri(uri, conf=self.conf)
self.store.delete(loc)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
def get_from_backend(uri, offset=0, chunk_size=None, context=None):
"""Yields chunks of data from backend specified by uri."""
loc = location.get_location_from_uri(uri, conf=CONF)
store = get_store_from_uri(uri)
return store.get(loc, offset=offset,
chunk_size=chunk_size,
context=context)
