def test_copy_to_swift(self): log = BufferLogger() # Confirm that files exist on disk where we expect to find them. for lfc in self.lfcs: path = swift.filesystem_path(lfc.id) self.assertTrue(os.path.exists(path)) # Copy all the files into Swift. swift.to_swift(log, remove_func=None) # Confirm that files exist on disk where we expect to find them. for lfc in self.lfcs: path = swift.filesystem_path(lfc.id) self.assertTrue(os.path.exists(path)) # Confirm all the files are also in Swift. swift_client = self.swift_fixture.connect() for lfc, contents in zip(self.lfcs, self.contents): container, name = swift.swift_location(lfc.id) headers, obj = swift_client.get_object(container, name) self.assertEqual(contents, obj, 'Did not round trip') # Running again does nothing, in particular does not reupload # the files to Swift. con_patch = patch.object(swift.swiftclient.Connection, 'put_object', side_effect=AssertionError('do not call')) with con_patch: swift.to_swift(log) # remove_func == None
def add_file(self, name, content, when=None, content_type='text/plain'): lfa_id = self.librarian_client.addFile(name=name, size=len(content), file=StringIO(content), contentType=content_type) if when is None: when = 0 # Very very old lfa = IStore(LibraryFileAlias).get(LibraryFileAlias, lfa_id) path = swift.filesystem_path(lfa.content.id) os.utime(path, (when, when)) return lfa_id
def test_move_to_swift(self): log = BufferLogger() # Confirm that files exist on disk where we expect to find them. for lfc in self.lfcs: path = swift.filesystem_path(lfc.id) self.assertTrue(os.path.exists(path)) # Migrate all the files into Swift. swift.to_swift(log, remove_func=os.unlink) # Confirm that all the files have gone from disk. for lfc in self.lfcs: self.assertFalse(os.path.exists(swift.filesystem_path(lfc.id))) # Confirm all the files are in Swift. swift_client = self.swift_fixture.connect() for lfc, contents in zip(self.lfcs, self.contents): container, name = swift.swift_location(lfc.id) headers, obj = swift_client.get_object(container, name) self.assertEqual(contents, obj, 'Did not round trip')
def test_largish_binary_files_from_swift(self): # Generate large blob, multiple of the chunk size. # Including null bytes for kicks. # A largish file is large enough that the HTTP upload needs # to be done in multiple chunks, but small enough that it is # stored in Swift as a single object. size = LibrarianStorage.CHUNK_SIZE * 50 self.assertTrue(size > 1024 * 1024) expected_content = ''.join(chr(i % 256) for i in range(0, size)) lfa_id = self.add_file('hello_bigboy.xls', expected_content) lfc = IStore(LibraryFileAlias).get(LibraryFileAlias, lfa_id).content # This data size is a multiple of our chunk size. self.assertEqual(0, len(expected_content) % LibrarianStorage.CHUNK_SIZE) # Data round trips when served from Swift. swift.to_swift(BufferLogger(), remove_func=os.unlink) self.assertFalse(os.path.exists(swift.filesystem_path(lfc.id))) lfa = self.librarian_client.getFileByAlias(lfa_id) self.assertEqual(expected_content, lfa.read())