def test_remove_members(self):
items = temp_archive.make_temp_item_list([
(self.native_filename('foo-1.2.3/fruits/apple.txt'), 'apple.txt'),
(self.native_filename('foo-1.2.3/fruits/durian.txt'),
'durian.txt'),
(self.native_filename('foo-1.2.3/fruits/kiwi.txt'), 'kiwi.txt'),
(self.native_filename('foo-1.2.3/.import/foo.txt'), 'foo.txt'),
(self.native_filename('foo-1.2.3/.import/bar.txt'), 'bar.txt'),
])
tmp_archive = temp_archive.make_temp_archive(items,
'zip',
delete=not self.DEBUG)
self.assertEqual([
'foo-1.2.3/.import/bar.txt',
'foo-1.2.3/.import/foo.txt',
'foo-1.2.3/fruits/apple.txt',
'foo-1.2.3/fruits/durian.txt',
'foo-1.2.3/fruits/kiwi.txt',
], archiver.members(tmp_archive))
archive_util.remove_members(tmp_archive, ['foo-1.2.3/.import'],
debug=self.DEBUG)
self.assertEqual([
'foo-1.2.3/fruits/apple.txt',
'foo-1.2.3/fruits/durian.txt',
'foo-1.2.3/fruits/kiwi.txt',
], archiver.members(tmp_archive))
def test_remove_members_matching_patterns(self):
items = temp_archive.make_temp_item_list([
(self.native_filename('foo-1.2.3/fruits/apple.txt'), 'apple.txt'),
(self.native_filename('foo-1.2.3/fruits/durian.txt'),
'durian.txt'),
(self.native_filename('foo-1.2.3/fruits/kiwi.txt'), 'kiwi.txt'),
(self.native_filename('foo-1.2.3/.import/foo.txt'), 'foo.txt'),
(self.native_filename('foo-1.2.3/.import/bar.txt'), 'bar.txt'),
(self.native_filename('foo-1.2.3/cheese/brie.jpg'), 'brie.jpg'),
(self.native_filename('foo-1.2.3/cheese/halumi.jpg'),
'halumi.jpg'),
(self.native_filename('foo-1.2.3/cheese/feta.jpg'), 'feta.jpg'),
])
tmp_archive = temp_archive.make_temp_archive(items,
'zip',
delete=not self.DEBUG)
archive_util.remove_members_matching_patterns(tmp_archive,
['notfound'],
debug=self.DEBUG)
self.assertEqual([
'foo-1.2.3/.import/bar.txt',
'foo-1.2.3/.import/foo.txt',
'foo-1.2.3/cheese/brie.jpg',
'foo-1.2.3/cheese/feta.jpg',
'foo-1.2.3/cheese/halumi.jpg',
'foo-1.2.3/fruits/apple.txt',
'foo-1.2.3/fruits/durian.txt',
'foo-1.2.3/fruits/kiwi.txt',
], archiver.members(tmp_archive))
tmp_archive = temp_archive.make_temp_archive(items,
'zip',
delete=not self.DEBUG)
archive_util.remove_members_matching_patterns(tmp_archive, ['*.txt'],
debug=self.DEBUG)
self.assertEqual([
'foo-1.2.3/cheese/brie.jpg',
'foo-1.2.3/cheese/feta.jpg',
'foo-1.2.3/cheese/halumi.jpg',
], archiver.members(tmp_archive))
tmp_archive = temp_archive.make_temp_archive(items,
'zip',
delete=not self.DEBUG)
archive_util.remove_members_matching_patterns(tmp_archive,
['*cheese*'],
debug=self.DEBUG)
self.assertEqual([
'foo-1.2.3/.import/bar.txt',
'foo-1.2.3/.import/foo.txt',
'foo-1.2.3/fruits/apple.txt',
'foo-1.2.3/fruits/durian.txt',
'foo-1.2.3/fruits/kiwi.txt',
], archiver.members(tmp_archive))
def test_create_git_tgz(self):
content = [
'file a/b/c/foo.txt "foo content" 755',
'file d/e/bar.txt "bar content" 644',
'dir baz "" 700',
]
r = git_temp_repo(content = content, prefix = 'test_archive_', debug = self.DEBUG)
commit = r.last_commit_hash(short_hash = True)
tmp_archive = self.make_temp_file(suffix = '.tgz')
prefix = 'foo-{}'.format(commit)
args = [
'archive',
'create_git',
'--root-dir', r.root,
'--format', 'tgz',
prefix,
commit,
tmp_archive,
]
rv = self.run_program(self._program, args)
self.assertEqual( 0, rv.exit_code )
self.assertEqual( 'tgz', archiver.format_name(tmp_archive) )
expected = [
'{}/a/b/c/foo.txt'.format(prefix),
'{}/d/e/bar.txt'.format(prefix),
]
actual = archiver.members(tmp_archive)
self.assert_filename_list_equal( expected, actual )
def find_and_unsplit(clazz, files, options=None):
check.check_string_seq(files)
check.check_file_split_options(options, allow_none=True)
info = clazz.find_and_unsplit_info(files, options=options)
for item in info.items:
item_target = item.target
options.blurber.blurb_verbose(
f'Unsplitting {item_target} - {len(item.files)} parts.')
tmp = temp_file.make_temp_file(prefix=path.basename(item_target),
dir=path.dirname(item_target))
clazz.unsplit_files(tmp, item.files)
if options.unzip:
if archiver.is_valid(tmp):
members = archiver.members(tmp)
num_members = len(members)
if num_members != 1:
options.blurber.blurb(
f'{item_target} archive should have exactly 1 member instead of {num_members}'
)
else:
archive_filename = members[0]
archive_tmp_dir = temp_file.make_temp_dir(
prefix=path.basename(archive_filename),
dir=path.dirname(item_target),
delete=False)
archiver.extract_all(tmp, archive_tmp_dir)
archive_tmp_file = path.join(archive_tmp_dir,
archive_filename)
assert path.exists(archive_tmp_file)
file_util.rename(archive_tmp_file, tmp)
file_util.remove(archive_tmp_dir)
item_target = path.join(path.dirname(item_target),
archive_filename)
target = None
if path.exists(item_target):
if file_util.files_are_the_same(tmp, item_target):
options.blurber.blurb(
f'{item_target} already exists and is the same')
file_util.remove(tmp)
else:
ts = time_util.timestamp(
delimiter='',
milliseconds=False,
when=options.existing_file_timestamp)
target = clazz._make_timestamp_filename(item_target, ts)
options.blurber.blurb(
f'{item_target} already exists but is different. Renaming to {target}'
)
else:
target = item_target
if target:
file_util.rename(tmp, target)
file_util.remove(item.files)
def xtest_tarball(self):
server = web_server_controller(self._tarball_web_server)
server.start()
port = server.address[1]
url = self._make_url(port, 'sources/foo/foo-1.2.3.tar.gz')
tmp = url_util.download_to_temp_file(url, basename='foo-1.2.3.tar.gz')
self.debug_spew_filename('tmp', tmp)
self.assertEqual(['apple.txt', 'orange.txt'], archiver.members(tmp))
self.assertTrue(file_mime.is_gzip(tmp))
file_util.remove(tmp)
url = self._make_url(port, 'sources/bar/bar-1.2.3.zip')
tmp = url_util.download_to_temp_file(url, basename='bar-1.2.3.zip')
self.debug_spew_filename('tmp', tmp)
self.assertEqual(['apple.txt', 'orange.txt'], archiver.members(tmp))
self.assertTrue(file_mime.is_zip(tmp))
file_util.remove(tmp)
server.stop()
def xtest_fail_next_request(self):
server = web_server_controller(self._tarball_web_server)
server.start()
port = server.address[1]
url = self._make_url(port, 'sources/foo/foo-1.2.3.tar.gz')
tmp = url_util.download_to_temp_file(url, basename='foo-1.2.3.tar.gz')
self.assertEqual(['apple.txt', 'orange.txt'], archiver.members(tmp))
file_util.remove(tmp)
server.fail_next_request(404)
with self.assertRaises((url_compat.HTTPError, RuntimeError)) as ctx:
url_util.download_to_temp_file(url, basename='foo-1.2.3.tar.gz')
tmp = url_util.download_to_temp_file(url, basename='foo-1.2.3.tar.gz')
self.assertEqual(['apple.txt', 'orange.txt'], archiver.members(tmp))
file_util.remove(tmp)
server.stop()
def test_create_with_exclude(self):
tmp_dir = temp_content.write_items_to_temp_dir([
'file a/b/c/foo.txt "foo content" 755',
'file d/e/bar.txt "bar content" 644',
'dir baz "" 700',
])
tmp_archive = self.make_temp_file(suffix = '.zip')
archiver.create(tmp_archive, tmp_dir, exclude = [ self.native_filename('d/e/bar.txt') ] )
self.assertEqual( [
'a/b/c/foo.txt',
], archiver.members(tmp_archive) )
def test_archive_local_repo(self):
tmp_repo = self._create_tmp_repo()
new_files = self._create_tmp_files(tmp_repo)
git.add(tmp_repo, new_files)
git.commit(tmp_repo, 'nomsg\n', '.')
tmp_archive = self.make_temp_file()
git.archive(tmp_repo, 'master', 'foo', tmp_archive)
self.assertEqual([
'foo-master/',
'foo-master/bar.txt',
'foo-master/foo.txt',
], archiver.members(tmp_archive))
def test_filenames_with_brackets_tar(self):
tmp_dir = temp_content.write_items_to_temp_dir([
'file .foo/bar-[baz].ext-kiwi.apple.lemon "something" 644',
], delete = not self.DEBUG)
tmp_archive = self.make_temp_file()
archiver.create(tmp_archive, tmp_dir, extension = 'tar.gz')
self.assertEqual( [
'.foo/bar-[baz].ext-kiwi.apple.lemon',
], archiver.members(tmp_archive) )
self.assertEqual( 'tgz', archiver.format_name(tmp_archive) )
self.assertEqual( '3fc9b689459d738f8c88a3a48aa9e33542016b7a4052e001aaa536fca74813cb',
archiver.member_checksum(tmp_archive, '.foo/bar-[baz].ext-kiwi.apple.lemon') )
def test_create_force_tar_gz(self):
tmp_dir = temp_content.write_items_to_temp_dir([
'file a/b/c/foo.txt "foo content" 755',
'file d/e/bar.txt "bar content" 644',
'dir baz "" 700',
])
tmp_archive = self.make_temp_file()
archiver.create(tmp_archive, tmp_dir, extension = 'tar.gz')
self.assertEqual( [
'a/b/c/foo.txt',
'd/e/bar.txt',
], archiver.members(tmp_archive) )
self.assertEqual( 'tgz', archiver.format_name(tmp_archive) )
def test_create_zip(self):
tmp_dir = temp_content.write_items_to_temp_dir([
'file a/b/c/foo.txt "foo content" 755',
'file d/e/bar.txt "bar content" 644',
'dir baz "" 700',
])
tmp_archive = self.make_temp_file(suffix = '.zip')
archiver.create(tmp_archive, tmp_dir)
self.assertEqual( [
'a/b/c/foo.txt',
'd/e/bar.txt',
], archiver.members(tmp_archive) )
self.assertEqual( 'zip', archiver.format_name(tmp_archive) )
def test_tarball_large(self):
server = web_server_controller(self._tarball_web_server)
server.start()
port = server.address[1]
url = self._make_url(port, 'sources/large/large-1.2.3.tar.gz')
tmp = url_util.download_to_temp_file(url,
basename='large-1.2.3.tar.gz')
self.debug_spew_filename('tmp', tmp)
self.assertEqual(['kiwi.bin'], archiver.members(tmp))
self.assertTrue(file_mime.is_gzip(tmp))
file_util.remove(tmp)
server.stop()
def test_archive_local_repo_untracked(self):
tmp_repo = self._create_tmp_repo()
new_files = self._create_tmp_files(tmp_repo)
git.add(tmp_repo, new_files)
git.commit(tmp_repo, 'nomsg\n', '.')
file_util.save(path.join(tmp_repo, 'kiwi.txt'),
content='this is kiwi.txt\n')
tmp_archive = self.make_temp_file()
git.archive(tmp_repo, 'master', 'foo', tmp_archive, untracked=True)
self.assertEqual([
'foo-master/',
'foo-master/bar.txt',
'foo-master/foo.txt',
'foo-master/kiwi.txt',
], archiver.members(tmp_archive))
def test_combine_conflicts_different_content_no_check(self):
a1 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('fruits/apple.txt'), 'apple.txt'),
(self.native_filename('fruits/durian.txt'), 'durian.txt'),
(self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'),
(self.native_filename('fruits/plum.txt'), '1plum.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a1-')
a2 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'),
(self.native_filename('fruits/melon.txt'), 'melon.txt'),
(self.native_filename('fruits/lemon.txt'), '1lemon.txt'),
(self.native_filename('fruits/dragonfruit.txt'),
'dragonfruit.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a2-')
a3 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('fruits/lemon.txt'), '2lemon.txt'),
(self.native_filename('fruits/strawberry.txt'), 'strawberry.txt'),
(self.native_filename('fruits/pear.txt'), 'pear.txt'),
(self.native_filename('fruits/plum.txt'), '2plum.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a3-')
tmp_archive = self.make_temp_file(suffix='.zip')
archive_util.combine([a1, a2, a3], tmp_archive)
self.assertEqual([
'fruits/apple.txt',
'fruits/dragonfruit.txt',
'fruits/durian.txt',
'fruits/kiwi.txt',
'fruits/lemon.txt',
'fruits/melon.txt',
'fruits/pear.txt',
'fruits/plum.txt',
'fruits/strawberry.txt',
], archiver.members(tmp_archive))
def test_transform(self):
tmp_dir = temp_content.write_items_to_temp_dir([
'file files/foo.txt "this is foo.txt" 644',
'file files/bar.txt "this is bar.txt" 644',
'file files/baz.txt "this is baz.txt" 644',
], delete = not self.DEBUG)
tmp_archive = self.make_temp_file(suffix = '.zip')
archiver.create(tmp_archive, tmp_dir)
from bes.archive.archive_operation_add_file import archive_operation_add_file
from bes.archive.archive_operation_remove_files import archive_operation_remove_files
operations = [
archive_operation_add_file('new/new_file.txt', 'this is new_file.txt', 0o0644),
archive_operation_remove_files([ 'files/foo.txt', 'files/bar.txt' ]),
]
archiver.transform(tmp_archive, operations)
self.assertEqual( [
'files/baz.txt',
'new/new_file.txt',
], archiver.members(tmp_archive) )
self.assertEqual( 'this is new_file.txt', archiver.extract_member_to_string(tmp_archive, 'new/new_file.txt', codec = 'utf8') )
def test_combine_with_base_dir(self):
a1 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('fruits/apple.txt'), 'apple.txt'),
(self.native_filename('fruits/durian.txt'), 'durian.txt'),
(self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a1-')
a2 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('fruits/melon.txt'), 'melon.txt'),
(self.native_filename('fruits/lemon.txt'), 'lemon.txt'),
(self.native_filename('fruits/dragonfruit.txt'),
'dragonfruit.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a2-')
a3 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([
(self.native_filename('fruits/strawberry.txt'), 'strawberry.txt'),
(self.native_filename('fruits/pear.txt'), 'pear.txt'),
(self.native_filename('fruits/plum.txt'), 'plum.txt'),
]),
'zip',
delete=not self.DEBUG,
prefix='a3-')
tmp_archive = self.make_temp_file(suffix='.zip')
archive_util.combine([a1, a2, a3], tmp_archive, base_dir='foo-1.2.3')
self.assertEqual([
'foo-1.2.3/fruits/apple.txt',
'foo-1.2.3/fruits/dragonfruit.txt',
'foo-1.2.3/fruits/durian.txt',
'foo-1.2.3/fruits/kiwi.txt',
'foo-1.2.3/fruits/lemon.txt',
'foo-1.2.3/fruits/melon.txt',
'foo-1.2.3/fruits/pear.txt',
'foo-1.2.3/fruits/plum.txt',
'foo-1.2.3/fruits/strawberry.txt',
], archiver.members(tmp_archive))
def _do_contents_one_archive(clazz, archive, print_archive):
for member in archiver.members(archive):
if print_archive:
print('{}:{}'.format(path.basename(archive), member))
else:
print(member)
