def test_copy_to_swift(self):
log = BufferLogger()
# Confirm that files exist on disk where we expect to find them.
for lfc in self.lfcs:
path = swift.filesystem_path(lfc.id)
self.assertTrue(os.path.exists(path))
# Copy all the files into Swift.
swift.to_swift(log, remove_func=None)
# Confirm that files exist on disk where we expect to find them.
for lfc in self.lfcs:
path = swift.filesystem_path(lfc.id)
self.assertTrue(os.path.exists(path))
# Confirm all the files are also in Swift.
swift_client = self.swift_fixture.connect()
for lfc, contents in zip(self.lfcs, self.contents):
container, name = swift.swift_location(lfc.id)
headers, obj = swift_client.get_object(container, name)
self.assertEqual(contents, obj, 'Did not round trip')
# Running again does nothing, in particular does not reupload
# the files to Swift.
con_patch = patch.object(swift.swiftclient.Connection,
'put_object',
side_effect=AssertionError('do not call'))
with con_patch:
swift.to_swift(log) # remove_func == None
def test_librarian_serves_from_swift(self):
log = BufferLogger()
# Move all the files into Swift and off the file system.
swift.to_swift(log, remove_func=os.unlink)
# Confirm we can still access the files from the Librarian.
for lfa_id, content in zip(self.lfa_ids, self.contents):
data = self.librarian_client.getFileByAlias(lfa_id).read()
self.assertEqual(content, data)
def test_large_file_to_swift(self):
# Generate a blob large enough that Swift requires us to store
# it as multiple objects plus a manifest.
size = LibrarianStorage.CHUNK_SIZE * 50
self.assertTrue(size > 1024 * 1024)
expected_content = ''.join(chr(i % 256) for i in range(0, size))
lfa_id = self.add_file('hello_bigboy.xls', expected_content)
lfa = IStore(LibraryFileAlias).get(LibraryFileAlias, lfa_id)
lfc = lfa.content
# We don't really want to upload a file >5GB to our mock Swift,
# so change the constant instead. Set it so we need 3 segments.
def _reset_max(val):
swift.MAX_SWIFT_OBJECT_SIZE = val
self.addCleanup(_reset_max, swift.MAX_SWIFT_OBJECT_SIZE)
swift.MAX_SWIFT_OBJECT_SIZE = int(size / 2) - 1
# Shove the file requiring multiple segments into Swift.
swift.to_swift(BufferLogger(), remove_func=None)
# As our mock Swift does not support multi-segment files,
# instead we examine it directly in Swift as best we can.
swift_client = self.swift_fixture.connect()
# The manifest exists. Unfortunately, we can't test that the
# magic manifest header is set correctly.
container, name = swift.swift_location(lfc.id)
headers, obj = swift_client.get_object(container, name)
self.assertEqual(obj, '')
# The segments we expect are all in their expected locations.
_, obj1 = swift_client.get_object(container, '{0}/0000'.format(name))
_, obj2 = swift_client.get_object(container, '{0}/0001'.format(name))
_, obj3 = swift_client.get_object(container, '{0}/0002'.format(name))
self.assertRaises(swiftclient.ClientException, swift.quiet_swiftclient,
swift_client.get_object, container,
'{0}/0003'.format(name))
# Our object round tripped
self.assertEqual(obj1 + obj2 + obj3, expected_content)
def test_largish_binary_files_from_swift(self):
# Generate large blob, multiple of the chunk size.
# Including null bytes for kicks.
# A largish file is large enough that the HTTP upload needs
# to be done in multiple chunks, but small enough that it is
# stored in Swift as a single object.
size = LibrarianStorage.CHUNK_SIZE * 50
self.assertTrue(size > 1024 * 1024)
expected_content = ''.join(chr(i % 256) for i in range(0, size))
lfa_id = self.add_file('hello_bigboy.xls', expected_content)
lfc = IStore(LibraryFileAlias).get(LibraryFileAlias, lfa_id).content
# This data size is a multiple of our chunk size.
self.assertEqual(0,
len(expected_content) % LibrarianStorage.CHUNK_SIZE)
# Data round trips when served from Swift.
swift.to_swift(BufferLogger(), remove_func=os.unlink)
self.assertFalse(os.path.exists(swift.filesystem_path(lfc.id)))
lfa = self.librarian_client.getFileByAlias(lfa_id)
self.assertEqual(expected_content, lfa.read())
def test_move_to_swift(self):
log = BufferLogger()
# Confirm that files exist on disk where we expect to find them.
for lfc in self.lfcs:
path = swift.filesystem_path(lfc.id)
self.assertTrue(os.path.exists(path))
# Migrate all the files into Swift.
swift.to_swift(log, remove_func=os.unlink)
# Confirm that all the files have gone from disk.
for lfc in self.lfcs:
self.assertFalse(os.path.exists(swift.filesystem_path(lfc.id)))
# Confirm all the files are in Swift.
swift_client = self.swift_fixture.connect()
for lfc, contents in zip(self.lfcs, self.contents):
container, name = swift.swift_location(lfc.id)
headers, obj = swift_client.get_object(container, name)
self.assertEqual(contents, obj, 'Did not round trip')