def test_copy_to_swift(self): log = BufferLogger() # Confirm that files exist on disk where we expect to find them. for lfc in self.lfcs: path = swift.filesystem_path(lfc.id) self.assertTrue(os.path.exists(path)) # Copy all the files into Swift. swift.to_swift(log, remove_func=None) # Confirm that files exist on disk where we expect to find them. for lfc in self.lfcs: path = swift.filesystem_path(lfc.id) self.assertTrue(os.path.exists(path)) # Confirm all the files are also in Swift. swift_client = self.swift_fixture.connect() for lfc, contents in zip(self.lfcs, self.contents): container, name = swift.swift_location(lfc.id) headers, obj = swift_client.get_object(container, name) self.assertEqual(contents, obj, 'Did not round trip') # Running again does nothing, in particular does not reupload # the files to Swift. con_patch = patch.object(swift.swiftclient.Connection, 'put_object', side_effect=AssertionError('do not call')) with con_patch: swift.to_swift(log) # remove_func == None
def test_librarian_serves_from_swift(self): log = BufferLogger() # Move all the files into Swift and off the file system. swift.to_swift(log, remove_func=os.unlink) # Confirm we can still access the files from the Librarian. for lfa_id, content in zip(self.lfa_ids, self.contents): data = self.librarian_client.getFileByAlias(lfa_id).read() self.assertEqual(content, data)
def test_large_file_to_swift(self): # Generate a blob large enough that Swift requires us to store # it as multiple objects plus a manifest. size = LibrarianStorage.CHUNK_SIZE * 50 self.assertTrue(size > 1024 * 1024) expected_content = ''.join(chr(i % 256) for i in range(0, size)) lfa_id = self.add_file('hello_bigboy.xls', expected_content) lfa = IStore(LibraryFileAlias).get(LibraryFileAlias, lfa_id) lfc = lfa.content # We don't really want to upload a file >5GB to our mock Swift, # so change the constant instead. Set it so we need 3 segments. def _reset_max(val): swift.MAX_SWIFT_OBJECT_SIZE = val self.addCleanup(_reset_max, swift.MAX_SWIFT_OBJECT_SIZE) swift.MAX_SWIFT_OBJECT_SIZE = int(size / 2) - 1 # Shove the file requiring multiple segments into Swift. swift.to_swift(BufferLogger(), remove_func=None) # As our mock Swift does not support multi-segment files, # instead we examine it directly in Swift as best we can. swift_client = self.swift_fixture.connect() # The manifest exists. Unfortunately, we can't test that the # magic manifest header is set correctly. container, name = swift.swift_location(lfc.id) headers, obj = swift_client.get_object(container, name) self.assertEqual(obj, '') # The segments we expect are all in their expected locations. _, obj1 = swift_client.get_object(container, '{0}/0000'.format(name)) _, obj2 = swift_client.get_object(container, '{0}/0001'.format(name)) _, obj3 = swift_client.get_object(container, '{0}/0002'.format(name)) self.assertRaises(swiftclient.ClientException, swift.quiet_swiftclient, swift_client.get_object, container, '{0}/0003'.format(name)) # Our object round tripped self.assertEqual(obj1 + obj2 + obj3, expected_content)
def test_largish_binary_files_from_swift(self): # Generate large blob, multiple of the chunk size. # Including null bytes for kicks. # A largish file is large enough that the HTTP upload needs # to be done in multiple chunks, but small enough that it is # stored in Swift as a single object. size = LibrarianStorage.CHUNK_SIZE * 50 self.assertTrue(size > 1024 * 1024) expected_content = ''.join(chr(i % 256) for i in range(0, size)) lfa_id = self.add_file('hello_bigboy.xls', expected_content) lfc = IStore(LibraryFileAlias).get(LibraryFileAlias, lfa_id).content # This data size is a multiple of our chunk size. self.assertEqual(0, len(expected_content) % LibrarianStorage.CHUNK_SIZE) # Data round trips when served from Swift. swift.to_swift(BufferLogger(), remove_func=os.unlink) self.assertFalse(os.path.exists(swift.filesystem_path(lfc.id))) lfa = self.librarian_client.getFileByAlias(lfa_id) self.assertEqual(expected_content, lfa.read())
def test_move_to_swift(self): log = BufferLogger() # Confirm that files exist on disk where we expect to find them. for lfc in self.lfcs: path = swift.filesystem_path(lfc.id) self.assertTrue(os.path.exists(path)) # Migrate all the files into Swift. swift.to_swift(log, remove_func=os.unlink) # Confirm that all the files have gone from disk. for lfc in self.lfcs: self.assertFalse(os.path.exists(swift.filesystem_path(lfc.id))) # Confirm all the files are in Swift. swift_client = self.swift_fixture.connect() for lfc, contents in zip(self.lfcs, self.contents): container, name = swift.swift_location(lfc.id) headers, obj = swift_client.get_object(container, name) self.assertEqual(contents, obj, 'Did not round trip')