def test_combine_conflicts_different_content_with_check(self): a1 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('fruits/apple.txt'), 'apple.txt'), (self.native_filename('fruits/durian.txt'), 'durian.txt'), (self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'), (self.native_filename('fruits/plum.txt'), '1plum.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a1-') a2 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'), (self.native_filename('fruits/melon.txt'), 'melon.txt'), (self.native_filename('fruits/lemon.txt'), '1lemon.txt'), (self.native_filename('fruits/dragonfruit.txt'), 'dragonfruit.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a2-') a3 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('fruits/lemon.txt'), '2lemon.txt'), (self.native_filename('fruits/strawberry.txt'), 'strawberry.txt'), (self.native_filename('fruits/pear.txt'), 'pear.txt'), (self.native_filename('fruits/plum.txt'), '2plum.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a3-') tmp_archive = self.make_temp_file(suffix='.zip') with self.assertRaises(RuntimeError) as ctx: archive_util.combine([a1, a2, a3], tmp_archive, check_content=True)
def test_remove_members_matching_patterns(self): items = temp_archive.make_temp_item_list([ (self.native_filename('foo-1.2.3/fruits/apple.txt'), 'apple.txt'), (self.native_filename('foo-1.2.3/fruits/durian.txt'), 'durian.txt'), (self.native_filename('foo-1.2.3/fruits/kiwi.txt'), 'kiwi.txt'), (self.native_filename('foo-1.2.3/.import/foo.txt'), 'foo.txt'), (self.native_filename('foo-1.2.3/.import/bar.txt'), 'bar.txt'), (self.native_filename('foo-1.2.3/cheese/brie.jpg'), 'brie.jpg'), (self.native_filename('foo-1.2.3/cheese/halumi.jpg'), 'halumi.jpg'), (self.native_filename('foo-1.2.3/cheese/feta.jpg'), 'feta.jpg'), ]) tmp_archive = temp_archive.make_temp_archive(items, 'zip', delete=not self.DEBUG) archive_util.remove_members_matching_patterns(tmp_archive, ['notfound'], debug=self.DEBUG) self.assertEqual([ 'foo-1.2.3/.import/bar.txt', 'foo-1.2.3/.import/foo.txt', 'foo-1.2.3/cheese/brie.jpg', 'foo-1.2.3/cheese/feta.jpg', 'foo-1.2.3/cheese/halumi.jpg', 'foo-1.2.3/fruits/apple.txt', 'foo-1.2.3/fruits/durian.txt', 'foo-1.2.3/fruits/kiwi.txt', ], archiver.members(tmp_archive)) tmp_archive = temp_archive.make_temp_archive(items, 'zip', delete=not self.DEBUG) archive_util.remove_members_matching_patterns(tmp_archive, ['*.txt'], debug=self.DEBUG) self.assertEqual([ 'foo-1.2.3/cheese/brie.jpg', 'foo-1.2.3/cheese/feta.jpg', 'foo-1.2.3/cheese/halumi.jpg', ], archiver.members(tmp_archive)) tmp_archive = temp_archive.make_temp_archive(items, 'zip', delete=not self.DEBUG) archive_util.remove_members_matching_patterns(tmp_archive, ['*cheese*'], debug=self.DEBUG) self.assertEqual([ 'foo-1.2.3/.import/bar.txt', 'foo-1.2.3/.import/foo.txt', 'foo-1.2.3/fruits/apple.txt', 'foo-1.2.3/fruits/durian.txt', 'foo-1.2.3/fruits/kiwi.txt', ], archiver.members(tmp_archive))
def test_file_is_valid(self): tmp_zip = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.ZIP) self.assertTrue( archive_zip.file_is_valid(tmp_zip) ) tmp_tar = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TAR) self.assertFalse( archive_zip.file_is_valid(tmp_tar) ) tmp_tgz = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TGZ) self.assertFalse( archive_zip.file_is_valid(tmp_tgz) ) tmp_tar_gz = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TAR_GZ) self.assertFalse( archive_zip.file_is_valid(tmp_tar_gz) ) self.assertFalse( archive_zip.file_is_valid(self.make_temp_file(content = 'junk\n')) )
def test_file_is_valid(self): tmp_zip = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.ZIP) self.assertTrue( archive_zip.file_is_valid(tmp_zip.filename) ) tmp_tar = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TAR) self.assertFalse( archive_zip.file_is_valid(tmp_tar.filename) ) tmp_tgz = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TGZ) self.assertFalse( archive_zip.file_is_valid(tmp_tgz.filename) ) tmp_tar_gz = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TAR_GZ) self.assertFalse( archive_zip.file_is_valid(tmp_tar_gz.filename) ) self.assertFalse( archive_zip.file_is_valid(temp_file.make_temp_file(content = 'junk\n')) )
def handle_request(self, environ, start_response): path_info = environ['PATH_INFO'] self.log_i('handle_request(%s)' % (path_info)) if path_info not in self._known_tarballs: return self.response_error(start_response, 404) extension = file_util.extension(path_info) if 'large' in path_info: items = [ temp_archive.item('kiwi.bin', content=self._make_large_content()), ] else: items = [ temp_archive.item('apple.txt', content='apple.txt\n'), temp_archive.item('orange.txt', content='orange.txt\n'), ] tmp_archive = temp_archive.make_temp_archive(items, extension) tmp_mime_type = file_mime.mime_type(tmp_archive) content = file_util.read(tmp_archive) headers = [ ('Content-Type', str(tmp_mime_type)), ('Content-Length', str(len(content))), ] result = self.response_success(start_response, 200, [content], headers) file_util.remove(tmp_archive) return result
def write_archive(self, filename, items): p = self.file_path(filename) if path.exists(p): raise IOError('already exists {}'.format(filename)) extension = file_util.extension(filename) tmp_archive = temp_archive.make_temp_archive(items, extension) file_util.rename(tmp_archive, p)
def test_remove_members(self): items = temp_archive.make_temp_item_list([ (self.native_filename('foo-1.2.3/fruits/apple.txt'), 'apple.txt'), (self.native_filename('foo-1.2.3/fruits/durian.txt'), 'durian.txt'), (self.native_filename('foo-1.2.3/fruits/kiwi.txt'), 'kiwi.txt'), (self.native_filename('foo-1.2.3/.import/foo.txt'), 'foo.txt'), (self.native_filename('foo-1.2.3/.import/bar.txt'), 'bar.txt'), ]) tmp_archive = temp_archive.make_temp_archive(items, 'zip', delete=not self.DEBUG) self.assertEqual([ 'foo-1.2.3/.import/bar.txt', 'foo-1.2.3/.import/foo.txt', 'foo-1.2.3/fruits/apple.txt', 'foo-1.2.3/fruits/durian.txt', 'foo-1.2.3/fruits/kiwi.txt', ], archiver.members(tmp_archive)) archive_util.remove_members(tmp_archive, ['foo-1.2.3/.import'], debug=self.DEBUG) self.assertEqual([ 'foo-1.2.3/fruits/apple.txt', 'foo-1.2.3/fruits/durian.txt', 'foo-1.2.3/fruits/kiwi.txt', ], archiver.members(tmp_archive))
def test_match_members(self): tmp_archive = temp_archive.make_temp_archive( temp_archive.make_temp_item_list([ (self.native_filename('fruits/apple.pdf'), 'apple.pdf'), (self.native_filename('fruits/durian.pdf'), 'durian.pdf'), (self.native_filename('fruits/plum.pdf'), 'plum.pdf'), (self.native_filename('cheese/brie.txt'), 'brie.txt'), (self.native_filename('cheese/cheddar.txt'), 'cheddar.txt'), (self.native_filename('cheese/fontina.txt'), 'fontina.txt'), ]), 'zip', delete=not self.DEBUG) self.assertEqual([ 'cheese/brie.txt', 'cheese/cheddar.txt', 'cheese/fontina.txt', 'fruits/apple.pdf', 'fruits/durian.pdf', 'fruits/plum.pdf', ], archive_util.match_members(tmp_archive, ['*'])) self.assertEqual([ 'cheese/brie.txt', 'cheese/cheddar.txt', 'cheese/fontina.txt', ], archive_util.match_members(tmp_archive, ['cheese*'])) self.assertEqual([ 'cheese/brie.txt', 'cheese/cheddar.txt', 'cheese/fontina.txt', ], archive_util.match_members(tmp_archive, ['*.txt'])) self.assertEqual([ 'fruits/apple.pdf', 'fruits/durian.pdf', 'fruits/plum.pdf', ], archive_util.match_members(tmp_archive, ['*.pdf']))
def _make_temp_archive(self, extension, items=None): items = items or [temp_archive.item('foo.txt', content='foo.txt\n')] ta = temp_archive.make_temp_archive(items, extension, delete=not self.DEBUG) if self.DEBUG: self.spew('temp_archive(%s): %s' % (extension, ta)) return ta
def test_combine_conflicts_different_content_no_check(self): a1 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('fruits/apple.txt'), 'apple.txt'), (self.native_filename('fruits/durian.txt'), 'durian.txt'), (self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'), (self.native_filename('fruits/plum.txt'), '1plum.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a1-') a2 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'), (self.native_filename('fruits/melon.txt'), 'melon.txt'), (self.native_filename('fruits/lemon.txt'), '1lemon.txt'), (self.native_filename('fruits/dragonfruit.txt'), 'dragonfruit.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a2-') a3 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('fruits/lemon.txt'), '2lemon.txt'), (self.native_filename('fruits/strawberry.txt'), 'strawberry.txt'), (self.native_filename('fruits/pear.txt'), 'pear.txt'), (self.native_filename('fruits/plum.txt'), '2plum.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a3-') tmp_archive = self.make_temp_file(suffix='.zip') archive_util.combine([a1, a2, a3], tmp_archive) self.assertEqual([ 'fruits/apple.txt', 'fruits/dragonfruit.txt', 'fruits/durian.txt', 'fruits/kiwi.txt', 'fruits/lemon.txt', 'fruits/melon.txt', 'fruits/pear.txt', 'fruits/plum.txt', 'fruits/strawberry.txt', ], archiver.members(tmp_archive))
def test_duplicate_members_with_conflicts(self): a1 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('foo-1.2.3/fruits/apple.txt'), 'apple.txt'), (self.native_filename('foo-1.2.3/fruits/durian.txt'), 'durian.txt'), (self.native_filename('foo-1.2.3/fruits/kiwi.txt'), 'kiwi.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a1-') a2 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('foo-1.2.3/fruits/apple.txt'), 'apple2.txt'), (self.native_filename('foo-1.2.3/fruits/durian.txt'), 'durian.txt'), (self.native_filename('foo-1.2.3/fruits/melon.txt'), 'melon.txt'), (self.native_filename('foo-1.2.3/wine/barolo.txt'), 'barolo.txt'), (self.native_filename('foo-1.2.3/cheese/brie.txt'), 'brie.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a2-') a3 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('foo-1.2.3/fruits/strawberry.txt'), 'strawberry.txt'), (self.native_filename('foo-1.2.3/fruits/blueberry.txt'), 'blueberry.txt'), (self.native_filename('foo-1.2.3/fruits/banana.txt'), 'banana.txt'), (self.native_filename('foo-1.2.3/wine/barolo.txt'), 'barolo.txt'), (self.native_filename('foo-1.2.3/cheese/brie.txt'), 'brie2.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a3-') self.assertEqual( { 'foo-1.2.3/cheese/brie.txt': {a2, a3}, 'foo-1.2.3/fruits/apple.txt': {a1, a2}, }, archive_util.duplicate_members([a1, a2, a3], only_content_conficts=True))
def test_find_archives(self): tmp_dir = temp_file.make_temp_dir() tmp_zip = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.ZIP) tmp_tar = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TAR) tmp_tgz = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TGZ) tmp_tar_gz = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TAR_GZ) file_util.copy(tmp_zip, path.join(tmp_dir, 'archives/zip/tmp_zip.zip')) file_util.copy(tmp_tar, path.join(tmp_dir, 'archives/tar/tmp_tar.tar')) file_util.copy(tmp_tgz, path.join(tmp_dir, 'archives/tgz/tmp_tgz.tgz')) file_util.copy(tmp_tar_gz, path.join(tmp_dir, 'archives/tar_gz/tmp_tar_gz.tar.gz')) file_util.save(path.join(tmp_dir, 'archives/zip/fake_zip.zip'), content = 'not a zip\n') file_util.save(path.join(tmp_dir, 'archives/tar/fake_tar.tar'), content = 'not a tar\n') file_util.save(path.join(tmp_dir, 'archives/tar_gz/fake_tar_gz.tar.gz'), content = 'not a tar.gz\n') self.assertEqual( self.native_filename_list([ 'archives/tar/tmp_tar.tar', 'archives/tar_gz/tmp_tar_gz.tar.gz', 'archives/tgz/tmp_tgz.tgz', 'archives/zip/tmp_zip.zip', ]), archiver.find_archives(tmp_dir) )
def test_combine_with_base_dir(self): a1 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('fruits/apple.txt'), 'apple.txt'), (self.native_filename('fruits/durian.txt'), 'durian.txt'), (self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a1-') a2 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('fruits/melon.txt'), 'melon.txt'), (self.native_filename('fruits/lemon.txt'), 'lemon.txt'), (self.native_filename('fruits/dragonfruit.txt'), 'dragonfruit.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a2-') a3 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('fruits/strawberry.txt'), 'strawberry.txt'), (self.native_filename('fruits/pear.txt'), 'pear.txt'), (self.native_filename('fruits/plum.txt'), 'plum.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a3-') tmp_archive = self.make_temp_file(suffix='.zip') archive_util.combine([a1, a2, a3], tmp_archive, base_dir='foo-1.2.3') self.assertEqual([ 'foo-1.2.3/fruits/apple.txt', 'foo-1.2.3/fruits/dragonfruit.txt', 'foo-1.2.3/fruits/durian.txt', 'foo-1.2.3/fruits/kiwi.txt', 'foo-1.2.3/fruits/lemon.txt', 'foo-1.2.3/fruits/melon.txt', 'foo-1.2.3/fruits/pear.txt', 'foo-1.2.3/fruits/plum.txt', 'foo-1.2.3/fruits/strawberry.txt', ], archiver.members(tmp_archive))
def test_member_checksums(self): a = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('foo-1.2.3/fruits/apple.txt'), 'apple.txt'), (self.native_filename('foo-1.2.3/fruits/durian.txt'), 'durian.txt'), (self.native_filename('foo-1.2.3/fruits/kiwi.txt'), 'kiwi.txt'), ]), 'zip', delete=not self.DEBUG) self.assertEqual( { 'foo-1.2.3/fruits/apple.txt': '7269b27861e2a5ba6947b6279bb5e66b23439d83a65a3c0cf529f5834ed2e7fb', 'foo-1.2.3/fruits/kiwi.txt': 'a7be44d9dda7e951298316b34ce84a1b2da8b5e0bead26118145bda4fbca9329', }, archive_util.member_checksums( a, ['foo-1.2.3/fruits/apple.txt', 'foo-1.2.3/fruits/kiwi.txt']))
def test_find_and_unsplit_with_unzip_and_multiple_members(self): tmp = temp_archive.make_temp_archive([ temp_archive.item('kiwi.mp4', filename = self.mp4_file), temp_archive.item('lemon.jpg', filename = self.jpg_file), ], 'zip') tmp_dir = self.make_temp_dir() files = file_split.split_file(tmp, int(file_util.size(tmp) / 3), output_directory = tmp_dir, zfill_length = 3) self.assertEqual( 4, len(files) ) items = [ temp_content('file', 'src/kiwi.mp4.zip.001', file_util.read(files[0]), 0o0644), temp_content('file', 'src/kiwi.mp4.zip.002', file_util.read(files[1]), 0o0644), temp_content('file', 'src/kiwi.mp4.zip.003', file_util.read(files[2]), 0o0644), temp_content('file', 'src/kiwi.mp4.zip.004', file_util.read(files[3]), 0o0644), ] t = self._find_and_unsplit_test(extra_content_items = items, unzip = True) self.assertEqual( [ 'kiwi.mp4.zip', ], t.src_files )
def test_split_file_basic(self): NUM_ITEMS = 10 CONTENT_SIZE = 1024 * 100 items = [] for i in range(0, NUM_ITEMS): arcname = 'item{}.txt'.format(i) item = temp_archive.item(arcname, content = self._make_content(CONTENT_SIZE)) items.append(item) tmp_archive = temp_archive.make_temp_archive(items, 'zip') files = file_split.split_file(tmp_archive, int(math.floor(file_util.size(tmp_archive) / 1))) unsplit_tmp_archive = self.make_temp_file() file_split.unsplit_files(unsplit_tmp_archive, files) self.assertEqual( file_util.checksum('sha256', tmp_archive), file_util.checksum('sha256', unsplit_tmp_archive) ) file_util.remove(files) files = file_split.split_file(tmp_archive, int(math.floor(file_util.size(tmp_archive) / 2))) unsplit_tmp_archive = self.make_temp_file() file_split.unsplit_files(unsplit_tmp_archive, files) self.assertEqual( file_util.checksum('sha256', tmp_archive), file_util.checksum('sha256', unsplit_tmp_archive) ) file_util.remove(files) files = file_split.split_file(tmp_archive, int(math.floor(file_util.size(tmp_archive) / 3))) unsplit_tmp_archive = self.make_temp_file() file_split.unsplit_files(unsplit_tmp_archive, files) self.assertEqual( file_util.checksum('sha256', tmp_archive), file_util.checksum('sha256', unsplit_tmp_archive) ) file_util.remove(files) files = file_split.split_file(tmp_archive, int(math.floor(file_util.size(tmp_archive) / 4))) unsplit_tmp_archive = self.make_temp_file() file_split.unsplit_files(unsplit_tmp_archive, files) self.assertEqual( file_util.checksum('sha256', tmp_archive), file_util.checksum('sha256', unsplit_tmp_archive) ) file_util.remove(files) files = file_split.split_file(tmp_archive, int(math.floor(file_util.size(tmp_archive) / 5))) unsplit_tmp_archive = self.make_temp_file() file_split.unsplit_files(unsplit_tmp_archive, files) self.assertEqual( file_util.checksum('sha256', tmp_archive), file_util.checksum('sha256', unsplit_tmp_archive) ) file_util.remove(files)
def test_members(self): assert self.default_archive_type tmp_tar = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], self.default_archive_type, delete = not self.DEBUG) self.assertEqual( [ 'foo.txt' ], self.make_archive(tmp_tar.filename).members )
def make_temp_archive_for_reading(self, items, archive_type = None): archive_type = archive_type or self.default_archive_type assert archive_type tmp_archive = temp_archive.make_temp_archive(items, archive_type, delete = not self.DEBUG) return self.make_archive(tmp_archive.filename)
def _make_temp_archive(self, extension, items = None): items = items or [ temp_archive.item('foo.txt', content = 'foo.txt\n') ] ta = temp_archive.make_temp_archive(items, extension, delete = not self.DEBUG) if self.DEBUG: self.spew('temp_archive(%s): %s' % (extension, ta)) return ta