def test_remove_members(self): items = temp_archive.make_temp_item_list([ (self.native_filename('foo-1.2.3/fruits/apple.txt'), 'apple.txt'), (self.native_filename('foo-1.2.3/fruits/durian.txt'), 'durian.txt'), (self.native_filename('foo-1.2.3/fruits/kiwi.txt'), 'kiwi.txt'), (self.native_filename('foo-1.2.3/.import/foo.txt'), 'foo.txt'), (self.native_filename('foo-1.2.3/.import/bar.txt'), 'bar.txt'), ]) tmp_archive = temp_archive.make_temp_archive(items, 'zip', delete=not self.DEBUG) self.assertEqual([ 'foo-1.2.3/.import/bar.txt', 'foo-1.2.3/.import/foo.txt', 'foo-1.2.3/fruits/apple.txt', 'foo-1.2.3/fruits/durian.txt', 'foo-1.2.3/fruits/kiwi.txt', ], archiver.members(tmp_archive)) archive_util.remove_members(tmp_archive, ['foo-1.2.3/.import'], debug=self.DEBUG) self.assertEqual([ 'foo-1.2.3/fruits/apple.txt', 'foo-1.2.3/fruits/durian.txt', 'foo-1.2.3/fruits/kiwi.txt', ], archiver.members(tmp_archive))
def test_remove_members_matching_patterns(self): items = temp_archive.make_temp_item_list([ (self.native_filename('foo-1.2.3/fruits/apple.txt'), 'apple.txt'), (self.native_filename('foo-1.2.3/fruits/durian.txt'), 'durian.txt'), (self.native_filename('foo-1.2.3/fruits/kiwi.txt'), 'kiwi.txt'), (self.native_filename('foo-1.2.3/.import/foo.txt'), 'foo.txt'), (self.native_filename('foo-1.2.3/.import/bar.txt'), 'bar.txt'), (self.native_filename('foo-1.2.3/cheese/brie.jpg'), 'brie.jpg'), (self.native_filename('foo-1.2.3/cheese/halumi.jpg'), 'halumi.jpg'), (self.native_filename('foo-1.2.3/cheese/feta.jpg'), 'feta.jpg'), ]) tmp_archive = temp_archive.make_temp_archive(items, 'zip', delete=not self.DEBUG) archive_util.remove_members_matching_patterns(tmp_archive, ['notfound'], debug=self.DEBUG) self.assertEqual([ 'foo-1.2.3/.import/bar.txt', 'foo-1.2.3/.import/foo.txt', 'foo-1.2.3/cheese/brie.jpg', 'foo-1.2.3/cheese/feta.jpg', 'foo-1.2.3/cheese/halumi.jpg', 'foo-1.2.3/fruits/apple.txt', 'foo-1.2.3/fruits/durian.txt', 'foo-1.2.3/fruits/kiwi.txt', ], archiver.members(tmp_archive)) tmp_archive = temp_archive.make_temp_archive(items, 'zip', delete=not self.DEBUG) archive_util.remove_members_matching_patterns(tmp_archive, ['*.txt'], debug=self.DEBUG) self.assertEqual([ 'foo-1.2.3/cheese/brie.jpg', 'foo-1.2.3/cheese/feta.jpg', 'foo-1.2.3/cheese/halumi.jpg', ], archiver.members(tmp_archive)) tmp_archive = temp_archive.make_temp_archive(items, 'zip', delete=not self.DEBUG) archive_util.remove_members_matching_patterns(tmp_archive, ['*cheese*'], debug=self.DEBUG) self.assertEqual([ 'foo-1.2.3/.import/bar.txt', 'foo-1.2.3/.import/foo.txt', 'foo-1.2.3/fruits/apple.txt', 'foo-1.2.3/fruits/durian.txt', 'foo-1.2.3/fruits/kiwi.txt', ], archiver.members(tmp_archive))
def test_create_git_tgz(self): content = [ 'file a/b/c/foo.txt "foo content" 755', 'file d/e/bar.txt "bar content" 644', 'dir baz "" 700', ] r = git_temp_repo(content = content, prefix = 'test_archive_', debug = self.DEBUG) commit = r.last_commit_hash(short_hash = True) tmp_archive = self.make_temp_file(suffix = '.tgz') prefix = 'foo-{}'.format(commit) args = [ 'archive', 'create_git', '--root-dir', r.root, '--format', 'tgz', prefix, commit, tmp_archive, ] rv = self.run_program(self._program, args) self.assertEqual( 0, rv.exit_code ) self.assertEqual( 'tgz', archiver.format_name(tmp_archive) ) expected = [ '{}/a/b/c/foo.txt'.format(prefix), '{}/d/e/bar.txt'.format(prefix), ] actual = archiver.members(tmp_archive) self.assert_filename_list_equal( expected, actual )
def find_and_unsplit(clazz, files, options=None): check.check_string_seq(files) check.check_file_split_options(options, allow_none=True) info = clazz.find_and_unsplit_info(files, options=options) for item in info.items: item_target = item.target options.blurber.blurb_verbose( f'Unsplitting {item_target} - {len(item.files)} parts.') tmp = temp_file.make_temp_file(prefix=path.basename(item_target), dir=path.dirname(item_target)) clazz.unsplit_files(tmp, item.files) if options.unzip: if archiver.is_valid(tmp): members = archiver.members(tmp) num_members = len(members) if num_members != 1: options.blurber.blurb( f'{item_target} archive should have exactly 1 member instead of {num_members}' ) else: archive_filename = members[0] archive_tmp_dir = temp_file.make_temp_dir( prefix=path.basename(archive_filename), dir=path.dirname(item_target), delete=False) archiver.extract_all(tmp, archive_tmp_dir) archive_tmp_file = path.join(archive_tmp_dir, archive_filename) assert path.exists(archive_tmp_file) file_util.rename(archive_tmp_file, tmp) file_util.remove(archive_tmp_dir) item_target = path.join(path.dirname(item_target), archive_filename) target = None if path.exists(item_target): if file_util.files_are_the_same(tmp, item_target): options.blurber.blurb( f'{item_target} already exists and is the same') file_util.remove(tmp) else: ts = time_util.timestamp( delimiter='', milliseconds=False, when=options.existing_file_timestamp) target = clazz._make_timestamp_filename(item_target, ts) options.blurber.blurb( f'{item_target} already exists but is different. Renaming to {target}' ) else: target = item_target if target: file_util.rename(tmp, target) file_util.remove(item.files)
def xtest_tarball(self): server = web_server_controller(self._tarball_web_server) server.start() port = server.address[1] url = self._make_url(port, 'sources/foo/foo-1.2.3.tar.gz') tmp = url_util.download_to_temp_file(url, basename='foo-1.2.3.tar.gz') self.debug_spew_filename('tmp', tmp) self.assertEqual(['apple.txt', 'orange.txt'], archiver.members(tmp)) self.assertTrue(file_mime.is_gzip(tmp)) file_util.remove(tmp) url = self._make_url(port, 'sources/bar/bar-1.2.3.zip') tmp = url_util.download_to_temp_file(url, basename='bar-1.2.3.zip') self.debug_spew_filename('tmp', tmp) self.assertEqual(['apple.txt', 'orange.txt'], archiver.members(tmp)) self.assertTrue(file_mime.is_zip(tmp)) file_util.remove(tmp) server.stop()
def xtest_fail_next_request(self): server = web_server_controller(self._tarball_web_server) server.start() port = server.address[1] url = self._make_url(port, 'sources/foo/foo-1.2.3.tar.gz') tmp = url_util.download_to_temp_file(url, basename='foo-1.2.3.tar.gz') self.assertEqual(['apple.txt', 'orange.txt'], archiver.members(tmp)) file_util.remove(tmp) server.fail_next_request(404) with self.assertRaises((url_compat.HTTPError, RuntimeError)) as ctx: url_util.download_to_temp_file(url, basename='foo-1.2.3.tar.gz') tmp = url_util.download_to_temp_file(url, basename='foo-1.2.3.tar.gz') self.assertEqual(['apple.txt', 'orange.txt'], archiver.members(tmp)) file_util.remove(tmp) server.stop()
def test_create_with_exclude(self): tmp_dir = temp_content.write_items_to_temp_dir([ 'file a/b/c/foo.txt "foo content" 755', 'file d/e/bar.txt "bar content" 644', 'dir baz "" 700', ]) tmp_archive = self.make_temp_file(suffix = '.zip') archiver.create(tmp_archive, tmp_dir, exclude = [ self.native_filename('d/e/bar.txt') ] ) self.assertEqual( [ 'a/b/c/foo.txt', ], archiver.members(tmp_archive) )
def test_archive_local_repo(self): tmp_repo = self._create_tmp_repo() new_files = self._create_tmp_files(tmp_repo) git.add(tmp_repo, new_files) git.commit(tmp_repo, 'nomsg\n', '.') tmp_archive = self.make_temp_file() git.archive(tmp_repo, 'master', 'foo', tmp_archive) self.assertEqual([ 'foo-master/', 'foo-master/bar.txt', 'foo-master/foo.txt', ], archiver.members(tmp_archive))
def test_filenames_with_brackets_tar(self): tmp_dir = temp_content.write_items_to_temp_dir([ 'file .foo/bar-[baz].ext-kiwi.apple.lemon "something" 644', ], delete = not self.DEBUG) tmp_archive = self.make_temp_file() archiver.create(tmp_archive, tmp_dir, extension = 'tar.gz') self.assertEqual( [ '.foo/bar-[baz].ext-kiwi.apple.lemon', ], archiver.members(tmp_archive) ) self.assertEqual( 'tgz', archiver.format_name(tmp_archive) ) self.assertEqual( '3fc9b689459d738f8c88a3a48aa9e33542016b7a4052e001aaa536fca74813cb', archiver.member_checksum(tmp_archive, '.foo/bar-[baz].ext-kiwi.apple.lemon') )
def test_create_force_tar_gz(self): tmp_dir = temp_content.write_items_to_temp_dir([ 'file a/b/c/foo.txt "foo content" 755', 'file d/e/bar.txt "bar content" 644', 'dir baz "" 700', ]) tmp_archive = self.make_temp_file() archiver.create(tmp_archive, tmp_dir, extension = 'tar.gz') self.assertEqual( [ 'a/b/c/foo.txt', 'd/e/bar.txt', ], archiver.members(tmp_archive) ) self.assertEqual( 'tgz', archiver.format_name(tmp_archive) )
def test_create_zip(self): tmp_dir = temp_content.write_items_to_temp_dir([ 'file a/b/c/foo.txt "foo content" 755', 'file d/e/bar.txt "bar content" 644', 'dir baz "" 700', ]) tmp_archive = self.make_temp_file(suffix = '.zip') archiver.create(tmp_archive, tmp_dir) self.assertEqual( [ 'a/b/c/foo.txt', 'd/e/bar.txt', ], archiver.members(tmp_archive) ) self.assertEqual( 'zip', archiver.format_name(tmp_archive) )
def test_tarball_large(self): server = web_server_controller(self._tarball_web_server) server.start() port = server.address[1] url = self._make_url(port, 'sources/large/large-1.2.3.tar.gz') tmp = url_util.download_to_temp_file(url, basename='large-1.2.3.tar.gz') self.debug_spew_filename('tmp', tmp) self.assertEqual(['kiwi.bin'], archiver.members(tmp)) self.assertTrue(file_mime.is_gzip(tmp)) file_util.remove(tmp) server.stop()
def test_archive_local_repo_untracked(self): tmp_repo = self._create_tmp_repo() new_files = self._create_tmp_files(tmp_repo) git.add(tmp_repo, new_files) git.commit(tmp_repo, 'nomsg\n', '.') file_util.save(path.join(tmp_repo, 'kiwi.txt'), content='this is kiwi.txt\n') tmp_archive = self.make_temp_file() git.archive(tmp_repo, 'master', 'foo', tmp_archive, untracked=True) self.assertEqual([ 'foo-master/', 'foo-master/bar.txt', 'foo-master/foo.txt', 'foo-master/kiwi.txt', ], archiver.members(tmp_archive))
def test_combine_conflicts_different_content_no_check(self): a1 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('fruits/apple.txt'), 'apple.txt'), (self.native_filename('fruits/durian.txt'), 'durian.txt'), (self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'), (self.native_filename('fruits/plum.txt'), '1plum.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a1-') a2 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'), (self.native_filename('fruits/melon.txt'), 'melon.txt'), (self.native_filename('fruits/lemon.txt'), '1lemon.txt'), (self.native_filename('fruits/dragonfruit.txt'), 'dragonfruit.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a2-') a3 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('fruits/lemon.txt'), '2lemon.txt'), (self.native_filename('fruits/strawberry.txt'), 'strawberry.txt'), (self.native_filename('fruits/pear.txt'), 'pear.txt'), (self.native_filename('fruits/plum.txt'), '2plum.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a3-') tmp_archive = self.make_temp_file(suffix='.zip') archive_util.combine([a1, a2, a3], tmp_archive) self.assertEqual([ 'fruits/apple.txt', 'fruits/dragonfruit.txt', 'fruits/durian.txt', 'fruits/kiwi.txt', 'fruits/lemon.txt', 'fruits/melon.txt', 'fruits/pear.txt', 'fruits/plum.txt', 'fruits/strawberry.txt', ], archiver.members(tmp_archive))
def test_transform(self): tmp_dir = temp_content.write_items_to_temp_dir([ 'file files/foo.txt "this is foo.txt" 644', 'file files/bar.txt "this is bar.txt" 644', 'file files/baz.txt "this is baz.txt" 644', ], delete = not self.DEBUG) tmp_archive = self.make_temp_file(suffix = '.zip') archiver.create(tmp_archive, tmp_dir) from bes.archive.archive_operation_add_file import archive_operation_add_file from bes.archive.archive_operation_remove_files import archive_operation_remove_files operations = [ archive_operation_add_file('new/new_file.txt', 'this is new_file.txt', 0o0644), archive_operation_remove_files([ 'files/foo.txt', 'files/bar.txt' ]), ] archiver.transform(tmp_archive, operations) self.assertEqual( [ 'files/baz.txt', 'new/new_file.txt', ], archiver.members(tmp_archive) ) self.assertEqual( 'this is new_file.txt', archiver.extract_member_to_string(tmp_archive, 'new/new_file.txt', codec = 'utf8') )
def test_combine_with_base_dir(self): a1 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('fruits/apple.txt'), 'apple.txt'), (self.native_filename('fruits/durian.txt'), 'durian.txt'), (self.native_filename('fruits/kiwi.txt'), 'kiwi.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a1-') a2 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('fruits/melon.txt'), 'melon.txt'), (self.native_filename('fruits/lemon.txt'), 'lemon.txt'), (self.native_filename('fruits/dragonfruit.txt'), 'dragonfruit.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a2-') a3 = temp_archive.make_temp_archive(temp_archive.make_temp_item_list([ (self.native_filename('fruits/strawberry.txt'), 'strawberry.txt'), (self.native_filename('fruits/pear.txt'), 'pear.txt'), (self.native_filename('fruits/plum.txt'), 'plum.txt'), ]), 'zip', delete=not self.DEBUG, prefix='a3-') tmp_archive = self.make_temp_file(suffix='.zip') archive_util.combine([a1, a2, a3], tmp_archive, base_dir='foo-1.2.3') self.assertEqual([ 'foo-1.2.3/fruits/apple.txt', 'foo-1.2.3/fruits/dragonfruit.txt', 'foo-1.2.3/fruits/durian.txt', 'foo-1.2.3/fruits/kiwi.txt', 'foo-1.2.3/fruits/lemon.txt', 'foo-1.2.3/fruits/melon.txt', 'foo-1.2.3/fruits/pear.txt', 'foo-1.2.3/fruits/plum.txt', 'foo-1.2.3/fruits/strawberry.txt', ], archiver.members(tmp_archive))
def _do_contents_one_archive(clazz, archive, print_archive): for member in archiver.members(archive): if print_archive: print('{}:{}'.format(path.basename(archive), member)) else: print(member)