def test_combine_conflicts_different_content_with_check(self):
a1 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('fruits/apple.txt'), 'apple.txt'),
(self.native_filename('fruits/durian.txt'), 'durian.txt'),
(self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'),
(self.native_filename('fruits/plum.txt'), '1plum.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a1-')
a2 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'),
(self.native_filename('fruits/melon.txt'), 'melon.txt'),
(self.native_filename('fruits/lemon.txt'), '1lemon.txt'),
(self.native_filename('fruits/dragonfruit.txt'),
'dragonfruit.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a2-')
a3 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('fruits/lemon.txt'), '2lemon.txt'),
(self.native_filename('fruits/strawberry.txt'), 'strawberry.txt'),
(self.native_filename('fruits/pear.txt'), 'pear.txt'),
(self.native_filename('fruits/plum.txt'), '2plum.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a3-')
tmp_archive = self.make_temp_file(suffix='.zip')
with self.assertRaises(RuntimeError) as ctx:
archive_util.combine([a1, a2, a3], tmp_archive, check_content=True)
def test_remove_members_matching_patterns(self):
items = temp_archive.make_temp_item_list([
(self.native_filename('foo-1.2.3/fruits/apple.txt'), 'apple.txt'),
(self.native_filename('foo-1.2.3/fruits/durian.txt'),
'durian.txt'),
(self.native_filename('foo-1.2.3/fruits/kiwi.txt'), 'kiwi.txt'),
(self.native_filename('foo-1.2.3/.import/foo.txt'), 'foo.txt'),
(self.native_filename('foo-1.2.3/.import/bar.txt'), 'bar.txt'),
(self.native_filename('foo-1.2.3/cheese/brie.jpg'), 'brie.jpg'),
(self.native_filename('foo-1.2.3/cheese/halumi.jpg'),
'halumi.jpg'),
(self.native_filename('foo-1.2.3/cheese/feta.jpg'), 'feta.jpg'),
])
tmp_archive = temp_archive.make_temp_archive(items,
'zip',
delete=not self.DEBUG)
archive_util.remove_members_matching_patterns(tmp_archive,
['notfound'],
debug=self.DEBUG)
self.assertEqual([
'foo-1.2.3/.import/bar.txt',
'foo-1.2.3/.import/foo.txt',
'foo-1.2.3/cheese/brie.jpg',
'foo-1.2.3/cheese/feta.jpg',
'foo-1.2.3/cheese/halumi.jpg',
'foo-1.2.3/fruits/apple.txt',
'foo-1.2.3/fruits/durian.txt',
'foo-1.2.3/fruits/kiwi.txt',
], archiver.members(tmp_archive))
tmp_archive = temp_archive.make_temp_archive(items,
'zip',
delete=not self.DEBUG)
archive_util.remove_members_matching_patterns(tmp_archive, ['*.txt'],
debug=self.DEBUG)
self.assertEqual([
'foo-1.2.3/cheese/brie.jpg',
'foo-1.2.3/cheese/feta.jpg',
'foo-1.2.3/cheese/halumi.jpg',
], archiver.members(tmp_archive))
tmp_archive = temp_archive.make_temp_archive(items,
'zip',
delete=not self.DEBUG)
archive_util.remove_members_matching_patterns(tmp_archive,
['*cheese*'],
debug=self.DEBUG)
self.assertEqual([
'foo-1.2.3/.import/bar.txt',
'foo-1.2.3/.import/foo.txt',
'foo-1.2.3/fruits/apple.txt',
'foo-1.2.3/fruits/durian.txt',
'foo-1.2.3/fruits/kiwi.txt',
], archiver.members(tmp_archive))
def test_file_is_valid(self):
tmp_zip = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.ZIP)
self.assertTrue( archive_zip.file_is_valid(tmp_zip) )
tmp_tar = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TAR)
self.assertFalse( archive_zip.file_is_valid(tmp_tar) )
tmp_tgz = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TGZ)
self.assertFalse( archive_zip.file_is_valid(tmp_tgz) )
tmp_tar_gz = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TAR_GZ)
self.assertFalse( archive_zip.file_is_valid(tmp_tar_gz) )
self.assertFalse( archive_zip.file_is_valid(self.make_temp_file(content = 'junk\n')) )
def test_file_is_valid(self):
tmp_zip = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.ZIP)
self.assertTrue( archive_zip.file_is_valid(tmp_zip.filename) )
tmp_tar = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TAR)
self.assertFalse( archive_zip.file_is_valid(tmp_tar.filename) )
tmp_tgz = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TGZ)
self.assertFalse( archive_zip.file_is_valid(tmp_tgz.filename) )
tmp_tar_gz = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TAR_GZ)
self.assertFalse( archive_zip.file_is_valid(tmp_tar_gz.filename) )
self.assertFalse( archive_zip.file_is_valid(temp_file.make_temp_file(content = 'junk\n')) )
def handle_request(self, environ, start_response):
path_info = environ['PATH_INFO']
self.log_i('handle_request(%s)' % (path_info))
if path_info not in self._known_tarballs:
return self.response_error(start_response, 404)
extension = file_util.extension(path_info)
if 'large' in path_info:
items = [
temp_archive.item('kiwi.bin',
content=self._make_large_content()),
]
else:
items = [
temp_archive.item('apple.txt', content='apple.txt\n'),
temp_archive.item('orange.txt', content='orange.txt\n'),
]
tmp_archive = temp_archive.make_temp_archive(items, extension)
tmp_mime_type = file_mime.mime_type(tmp_archive)
content = file_util.read(tmp_archive)
headers = [
('Content-Type', str(tmp_mime_type)),
('Content-Length', str(len(content))),
]
result = self.response_success(start_response, 200, [content],
headers)
file_util.remove(tmp_archive)
return result
def write_archive(self, filename, items):
p = self.file_path(filename)
if path.exists(p):
raise IOError('already exists {}'.format(filename))
extension = file_util.extension(filename)
tmp_archive = temp_archive.make_temp_archive(items, extension)
file_util.rename(tmp_archive, p)
def test_remove_members(self):
items = temp_archive.make_temp_item_list([
(self.native_filename('foo-1.2.3/fruits/apple.txt'), 'apple.txt'),
(self.native_filename('foo-1.2.3/fruits/durian.txt'),
'durian.txt'),
(self.native_filename('foo-1.2.3/fruits/kiwi.txt'), 'kiwi.txt'),
(self.native_filename('foo-1.2.3/.import/foo.txt'), 'foo.txt'),
(self.native_filename('foo-1.2.3/.import/bar.txt'), 'bar.txt'),
])
tmp_archive = temp_archive.make_temp_archive(items,
'zip',
delete=not self.DEBUG)
self.assertEqual([
'foo-1.2.3/.import/bar.txt',
'foo-1.2.3/.import/foo.txt',
'foo-1.2.3/fruits/apple.txt',
'foo-1.2.3/fruits/durian.txt',
'foo-1.2.3/fruits/kiwi.txt',
], archiver.members(tmp_archive))
archive_util.remove_members(tmp_archive, ['foo-1.2.3/.import'],
debug=self.DEBUG)
self.assertEqual([
'foo-1.2.3/fruits/apple.txt',
'foo-1.2.3/fruits/durian.txt',
'foo-1.2.3/fruits/kiwi.txt',
], archiver.members(tmp_archive))
def test_match_members(self):
tmp_archive = temp_archive.make_temp_archive(
temp_archive.make_temp_item_list([
(self.native_filename('fruits/apple.pdf'), 'apple.pdf'),
(self.native_filename('fruits/durian.pdf'), 'durian.pdf'),
(self.native_filename('fruits/plum.pdf'), 'plum.pdf'),
(self.native_filename('cheese/brie.txt'), 'brie.txt'),
(self.native_filename('cheese/cheddar.txt'), 'cheddar.txt'),
(self.native_filename('cheese/fontina.txt'), 'fontina.txt'),
]),
'zip',
delete=not self.DEBUG)
self.assertEqual([
'cheese/brie.txt',
'cheese/cheddar.txt',
'cheese/fontina.txt',
'fruits/apple.pdf',
'fruits/durian.pdf',
'fruits/plum.pdf',
], archive_util.match_members(tmp_archive, ['*']))
self.assertEqual([
'cheese/brie.txt',
'cheese/cheddar.txt',
'cheese/fontina.txt',
], archive_util.match_members(tmp_archive, ['cheese*']))
self.assertEqual([
'cheese/brie.txt',
'cheese/cheddar.txt',
'cheese/fontina.txt',
], archive_util.match_members(tmp_archive, ['*.txt']))
self.assertEqual([
'fruits/apple.pdf',
'fruits/durian.pdf',
'fruits/plum.pdf',
], archive_util.match_members(tmp_archive, ['*.pdf']))
def _make_temp_archive(self, extension, items=None):
items = items or [temp_archive.item('foo.txt', content='foo.txt\n')]
ta = temp_archive.make_temp_archive(items,
extension,
delete=not self.DEBUG)
if self.DEBUG:
self.spew('temp_archive(%s): %s' % (extension, ta))
return ta
def test_combine_conflicts_different_content_no_check(self):
a1 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('fruits/apple.txt'), 'apple.txt'),
(self.native_filename('fruits/durian.txt'), 'durian.txt'),
(self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'),
(self.native_filename('fruits/plum.txt'), '1plum.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a1-')
a2 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'),
(self.native_filename('fruits/melon.txt'), 'melon.txt'),
(self.native_filename('fruits/lemon.txt'), '1lemon.txt'),
(self.native_filename('fruits/dragonfruit.txt'),
'dragonfruit.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a2-')
a3 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('fruits/lemon.txt'), '2lemon.txt'),
(self.native_filename('fruits/strawberry.txt'), 'strawberry.txt'),
(self.native_filename('fruits/pear.txt'), 'pear.txt'),
(self.native_filename('fruits/plum.txt'), '2plum.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a3-')
tmp_archive = self.make_temp_file(suffix='.zip')
archive_util.combine([a1, a2, a3], tmp_archive)
self.assertEqual([
'fruits/apple.txt',
'fruits/dragonfruit.txt',
'fruits/durian.txt',
'fruits/kiwi.txt',
'fruits/lemon.txt',
'fruits/melon.txt',
'fruits/pear.txt',
'fruits/plum.txt',
'fruits/strawberry.txt',
], archiver.members(tmp_archive))
def test_duplicate_members_with_conflicts(self):
a1 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('foo-1.2.3/fruits/apple.txt'), 'apple.txt'),
(self.native_filename('foo-1.2.3/fruits/durian.txt'),
'durian.txt'),
(self.native_filename('foo-1.2.3/fruits/kiwi.txt'), 'kiwi.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a1-')
a2 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('foo-1.2.3/fruits/apple.txt'), 'apple2.txt'),
(self.native_filename('foo-1.2.3/fruits/durian.txt'),
'durian.txt'),
(self.native_filename('foo-1.2.3/fruits/melon.txt'), 'melon.txt'),
(self.native_filename('foo-1.2.3/wine/barolo.txt'), 'barolo.txt'),
(self.native_filename('foo-1.2.3/cheese/brie.txt'), 'brie.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a2-')
a3 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('foo-1.2.3/fruits/strawberry.txt'),
'strawberry.txt'),
(self.native_filename('foo-1.2.3/fruits/blueberry.txt'),
'blueberry.txt'),
(self.native_filename('foo-1.2.3/fruits/banana.txt'),
'banana.txt'),
(self.native_filename('foo-1.2.3/wine/barolo.txt'), 'barolo.txt'),
(self.native_filename('foo-1.2.3/cheese/brie.txt'), 'brie2.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a3-')
self.assertEqual(
{
'foo-1.2.3/cheese/brie.txt': {a2, a3},
'foo-1.2.3/fruits/apple.txt': {a1, a2},
},
archive_util.duplicate_members([a1, a2, a3],
only_content_conficts=True))
def test_find_archives(self):
tmp_dir = temp_file.make_temp_dir()
tmp_zip = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.ZIP)
tmp_tar = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TAR)
tmp_tgz = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TGZ)
tmp_tar_gz = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], archive_extension.TAR_GZ)
file_util.copy(tmp_zip, path.join(tmp_dir, 'archives/zip/tmp_zip.zip'))
file_util.copy(tmp_tar, path.join(tmp_dir, 'archives/tar/tmp_tar.tar'))
file_util.copy(tmp_tgz, path.join(tmp_dir, 'archives/tgz/tmp_tgz.tgz'))
file_util.copy(tmp_tar_gz, path.join(tmp_dir, 'archives/tar_gz/tmp_tar_gz.tar.gz'))
file_util.save(path.join(tmp_dir, 'archives/zip/fake_zip.zip'), content = 'not a zip\n')
file_util.save(path.join(tmp_dir, 'archives/tar/fake_tar.tar'), content = 'not a tar\n')
file_util.save(path.join(tmp_dir, 'archives/tar_gz/fake_tar_gz.tar.gz'), content = 'not a tar.gz\n')
self.assertEqual( self.native_filename_list([
'archives/tar/tmp_tar.tar',
'archives/tar_gz/tmp_tar_gz.tar.gz',
'archives/tgz/tmp_tgz.tgz',
'archives/zip/tmp_zip.zip',
]), archiver.find_archives(tmp_dir) )
def test_combine_with_base_dir(self):
a1 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('fruits/apple.txt'), 'apple.txt'),
(self.native_filename('fruits/durian.txt'), 'durian.txt'),
(self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a1-')
a2 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('fruits/melon.txt'), 'melon.txt'),
(self.native_filename('fruits/lemon.txt'), 'lemon.txt'),
(self.native_filename('fruits/dragonfruit.txt'),
'dragonfruit.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a2-')
a3 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('fruits/strawberry.txt'), 'strawberry.txt'),
(self.native_filename('fruits/pear.txt'), 'pear.txt'),
(self.native_filename('fruits/plum.txt'), 'plum.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a3-')
tmp_archive = self.make_temp_file(suffix='.zip')
archive_util.combine([a1, a2, a3], tmp_archive, base_dir='foo-1.2.3')
self.assertEqual([
'foo-1.2.3/fruits/apple.txt',
'foo-1.2.3/fruits/dragonfruit.txt',
'foo-1.2.3/fruits/durian.txt',
'foo-1.2.3/fruits/kiwi.txt',
'foo-1.2.3/fruits/lemon.txt',
'foo-1.2.3/fruits/melon.txt',
'foo-1.2.3/fruits/pear.txt',
'foo-1.2.3/fruits/plum.txt',
'foo-1.2.3/fruits/strawberry.txt',
], archiver.members(tmp_archive))
def test_member_checksums(self):
a = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('foo-1.2.3/fruits/apple.txt'), 'apple.txt'),
(self.native_filename('foo-1.2.3/fruits/durian.txt'),
'durian.txt'),
(self.native_filename('foo-1.2.3/fruits/kiwi.txt'), 'kiwi.txt'),
]),
'zip',
delete=not self.DEBUG)
self.assertEqual(
{
'foo-1.2.3/fruits/apple.txt':
'7269b27861e2a5ba6947b6279bb5e66b23439d83a65a3c0cf529f5834ed2e7fb',
'foo-1.2.3/fruits/kiwi.txt':
'a7be44d9dda7e951298316b34ce84a1b2da8b5e0bead26118145bda4fbca9329',
},
archive_util.member_checksums(
a,
['foo-1.2.3/fruits/apple.txt', 'foo-1.2.3/fruits/kiwi.txt']))
def test_find_and_unsplit_with_unzip_and_multiple_members(self):
tmp = temp_archive.make_temp_archive([
temp_archive.item('kiwi.mp4', filename = self.mp4_file),
temp_archive.item('lemon.jpg', filename = self.jpg_file),
], 'zip')
tmp_dir = self.make_temp_dir()
files = file_split.split_file(tmp, int(file_util.size(tmp) / 3),
output_directory = tmp_dir,
zfill_length = 3)
self.assertEqual( 4, len(files) )
items = [
temp_content('file', 'src/kiwi.mp4.zip.001', file_util.read(files[0]), 0o0644),
temp_content('file', 'src/kiwi.mp4.zip.002', file_util.read(files[1]), 0o0644),
temp_content('file', 'src/kiwi.mp4.zip.003', file_util.read(files[2]), 0o0644),
temp_content('file', 'src/kiwi.mp4.zip.004', file_util.read(files[3]), 0o0644),
]
t = self._find_and_unsplit_test(extra_content_items = items,
unzip = True)
self.assertEqual( [
'kiwi.mp4.zip',
], t.src_files )
def test_split_file_basic(self):
NUM_ITEMS = 10
CONTENT_SIZE = 1024 * 100
items = []
for i in range(0, NUM_ITEMS):
arcname = 'item{}.txt'.format(i)
item = temp_archive.item(arcname, content = self._make_content(CONTENT_SIZE))
items.append(item)
tmp_archive = temp_archive.make_temp_archive(items, 'zip')
files = file_split.split_file(tmp_archive, int(math.floor(file_util.size(tmp_archive) / 1)))
unsplit_tmp_archive = self.make_temp_file()
file_split.unsplit_files(unsplit_tmp_archive, files)
self.assertEqual( file_util.checksum('sha256', tmp_archive), file_util.checksum('sha256', unsplit_tmp_archive) )
file_util.remove(files)
files = file_split.split_file(tmp_archive, int(math.floor(file_util.size(tmp_archive) / 2)))
unsplit_tmp_archive = self.make_temp_file()
file_split.unsplit_files(unsplit_tmp_archive, files)
self.assertEqual( file_util.checksum('sha256', tmp_archive), file_util.checksum('sha256', unsplit_tmp_archive) )
file_util.remove(files)
files = file_split.split_file(tmp_archive, int(math.floor(file_util.size(tmp_archive) / 3)))
unsplit_tmp_archive = self.make_temp_file()
file_split.unsplit_files(unsplit_tmp_archive, files)
self.assertEqual( file_util.checksum('sha256', tmp_archive), file_util.checksum('sha256', unsplit_tmp_archive) )
file_util.remove(files)
files = file_split.split_file(tmp_archive, int(math.floor(file_util.size(tmp_archive) / 4)))
unsplit_tmp_archive = self.make_temp_file()
file_split.unsplit_files(unsplit_tmp_archive, files)
self.assertEqual( file_util.checksum('sha256', tmp_archive), file_util.checksum('sha256', unsplit_tmp_archive) )
file_util.remove(files)
files = file_split.split_file(tmp_archive, int(math.floor(file_util.size(tmp_archive) / 5)))
unsplit_tmp_archive = self.make_temp_file()
file_split.unsplit_files(unsplit_tmp_archive, files)
self.assertEqual( file_util.checksum('sha256', tmp_archive), file_util.checksum('sha256', unsplit_tmp_archive) )
file_util.remove(files)
def test_members(self):
assert self.default_archive_type
tmp_tar = temp_archive.make_temp_archive([ temp_archive.item('foo.txt', content = 'foo.txt\n') ], self.default_archive_type, delete = not self.DEBUG)
self.assertEqual( [ 'foo.txt' ], self.make_archive(tmp_tar.filename).members )
def make_temp_archive_for_reading(self, items, archive_type = None): archive_type = archive_type or self.default_archive_type assert archive_type tmp_archive = temp_archive.make_temp_archive(items, archive_type, delete = not self.DEBUG) return self.make_archive(tmp_archive.filename)
def _make_temp_archive(self, extension, items = None):
items = items or [ temp_archive.item('foo.txt', content = 'foo.txt\n') ]
ta = temp_archive.make_temp_archive(items, extension, delete = not self.DEBUG)
if self.DEBUG:
self.spew('temp_archive(%s): %s' % (extension, ta))
return ta
