def test_resolve_one_level(self): tmp1 = self.make_temp_file(suffix = '-one') tmp2 = self.make_temp_file(suffix = '-two') file_util.remove(tmp2) os.symlink(tmp1, tmp2) self.assertEqual( tmp1, file_symlink.resolve(tmp1) ) self.assertEqual( tmp1, file_symlink.resolve(tmp2) )
def test_is_broken_false(self): tmp1 = tempfile.NamedTemporaryFile() tmp2 = tempfile.NamedTemporaryFile() file_util.remove(tmp1.name) os.symlink(tmp2.name, tmp1.name) self.assertEqual( True, path.islink(tmp1.name) ) self.assertEqual( False, file_symlink.is_broken(tmp1.name) )
def run_script(clazz, exe, script, args=None): 'Run the script and return the result.' check.check_string(exe) check.check_string(script) check.check_string_seq(args, allow_none=True) args = list(args or []) tmp_script = temp_file.make_temp_file(content=script, suffix='.py') cmd = [exe, tmp_script] + args clazz._log.log_d('run_script: exe={} cmd={}'.format(exe, cmd)) clazz._log.log_d('run_script: script=\n{}\n'.format(script)) try: output_bytes = subprocess.check_output(cmd, stderr=subprocess.STDOUT) exit_code = 0 clazz._log.log_d('run_script: success') except subprocess.CalledProcessError as ex: clazz._log.log_d('run_script: caught: {}'.format(str(ex))) output_bytes = ex.output exit_code = ex.returncode clazz._log.log_d('run_script: failed') finally: file_util.remove(tmp_script) output = codecs.decode(output_bytes, 'utf-8').strip() clazz._log.log_d('run_script: exit_code={} output="{}"') return clazz._run_script_result(exit_code, output)
def _do_set_credentials(clazz, username, password, print_error): expect_script = clazz._EXPECT_SCRIPT_TEMPLATE.format(username = username, password = password) # Remove ~/.vmrestCfg if corrupt clazz._vmrest_config_remove_corrupt() # Make sure there no stale vmrest processes vmware_util.killall_vmrest() tmp_script = temp_file.make_temp_file(suffix = '.expect', perm = 0o755, content = expect_script, delete = False) try: cmd = [ 'expect', '-d', tmp_script ] rv = execute.execute(cmd, raise_error = False, shell = False) clazz._log.log_d('_do_set_credentials: exit_code={} stdout="{}" stderr="{}"'.format(rv.exit_code, rv.stdout, rv.stderr)) if rv.exit_code != 0: raise vmware_error(clazz._EXPECT_SCRIPT_MSG_MAP[rv.exit_code]) finally: file_util.remove(tmp_script) if clazz._vmrest_config_is_corrupt(): msg = 'Failed to set credentials. Corrupt {}'.format(clazz._VMREST_CONFIG_FILENAME) if print_error: clazz._log.log_i(msg) raise vmware_error(msg)
def test_check_dir_symlink_success(self): tmp_dir = self.make_temp_dir() d = self.make_temp_dir(dir = tmp_dir) l = self.make_temp_file(content = 'link', dir = tmp_dir) file_symlink.symlink(d, l) self.assertEqual( d, file_check.check_dir(l) ) file_util.remove(tmp_dir)
def handle_request(self, environ, start_response): path_info = environ['PATH_INFO'] self.log_i('handle_request(%s)' % (path_info)) if path_info not in self._known_tarballs: return self.response_error(start_response, 404) extension = file_util.extension(path_info) if 'large' in path_info: items = [ temp_archive.item('kiwi.bin', content=self._make_large_content()), ] else: items = [ temp_archive.item('apple.txt', content='apple.txt\n'), temp_archive.item('orange.txt', content='orange.txt\n'), ] tmp_archive = temp_archive.make_temp_archive(items, extension) tmp_mime_type = file_mime.mime_type(tmp_archive) content = file_util.read(tmp_archive) headers = [ ('Content-Type', str(tmp_mime_type)), ('Content-Length', str(len(content))), ] result = self.response_success(start_response, 200, [content], headers) file_util.remove(tmp_archive) return result
def __exit__(self, type, value, traceback): if not path.isdir(self._where): raise IOError('Directory not found: "{}"'.format(self._where)) after = set(dir_util.list(self._where)) diff = after - self._before if diff: file_util.remove(list(diff))
def test_check_file_symlink_broken(self): f = self.make_temp_file(content = 'kiwi') l = self.make_temp_file(content = 'link') file_symlink.symlink(f, l) file_util.remove(f) with self.assertRaises(IOError) as ctx: file_check.check_file(l)
def backup(self, output_archive): 'Backup image to an gzipped tarball.' tmp_tar = temp_file.make_temp_file(suffix='.tar') args = ['save', self.tagged_repository, '-o', tmp_tar] docker_exe.call_docker(args, non_blocking=False) compressed_file.compress(tmp_tar, output_archive) file_util.remove(tmp_tar)
def repo_run_operation(clazz, address, operation, commit_message, options=None): check.check_string(address) check.check_callable(operation) check.check_string(commit_message) check.check_git_repo_operation_options(options, allow_none=True) options = options or git_repo_operation_options() if options.verbose: print('repo_run_operation: cloning {}'.format(address)) repo = clazz.clone_to_temp_dir(address, options=options, debug=options.debug) if options.debug: msg = 'repo_run_operation: repo.root={}'.format(repo.root) print(msg) clazz._LOG.log_d(msg) if options.dry_run: print('DRY_RUN: would execute {} on {} in {}'.format( operation, address, repo.root)) else: if options.verbose: print('repo_run_operation: executing {} in cwd={}'.format( operation, repo.root)) repo.operation_with_reset( operation, commit_message, num_tries=options.num_tries, retry_wait_seconds=options.retry_wait_seconds, files_to_commit=options.files_to_commit) file_util.remove(repo.root)
def _do_download_with_temp_ssh(clazz, address, revision, output_filename, base_name, download_options, parsed_address, ssh_credentials): domain_name = parsed_address.service clazz._log.log_d('_do_download_with_temp_ssh: domain_name={}'.format(domain_name)) temp_home = temp_file.make_temp_dir(delete = not download_options.debug) if download_options.debug: print('temp_home={}'.format(temp_home)) temp_ssh_dir = path.join(temp_home, '.ssh') clazz._log.log_d('_do_download_with_temp_ssh: temp_ssh_dir={}'.format(temp_ssh_dir)) cm = ssh_config_manager(temp_ssh_dir) installed = cm.install_key_pair_for_host(ssh_credentials.public_key, ssh_credentials.private_key, domain_name, username = ssh_credentials.username, host_key_type = ssh_credentials.host_key_type, include_ip_address = True, include_comment = True) # use a temporary HOME with custom git and ssh config just for this download with env_override.temp_home(use_temp_home = temp_home) as env: git_config.set_identity('test', '*****@*****.**') ssh_config_file = path.join(temp_ssh_dir, 'config') ssh_command = 'ssh -F {}'.format(ssh_config_file) git_config.set_value('core.sshCommand', ssh_command) clazz._do_download(address, revision, output_filename, base_name, download_options) file_util.remove(temp_home)
def test_check_dir_symlink_broken(self): d = self.make_temp_dir() l = self.make_temp_file(content = 'link') file_symlink.symlink(d, l) file_util.remove(d) with self.assertRaises(IOError) as ctx: file_check.check_dir(l)
def test_copy_tree_spaces_in_filenames(self): self.maxDiff = None src_tmp_dir = self.make_temp_dir(prefix='src-', suffix='-has 2 spaces-') dst_tmp_dir = self.make_temp_dir(prefix='dst-', suffix='-has 2 spaces-') file_util.remove(dst_tmp_dir) with tarfile.open(self.data_path('test.tar'), mode='r') as f: f.extractall(path=src_tmp_dir) file_copy.copy_tree(src_tmp_dir, dst_tmp_dir) expected_files = [ self.native_filename('1'), self.native_filename('1/2'), self.native_filename('1/2/3'), self.native_filename('1/2/3/4'), self.native_filename('1/2/3/4/5'), self.native_filename('1/2/3/4/5/apple.txt'), self.native_filename('1/2/3/4/5/kiwi.txt'), self.native_filename('bar.txt'), self.native_filename('empty'), self.native_filename('foo.txt'), self.native_filename('kiwi_link.txt'), ] actual_files = file_find.find(dst_tmp_dir, file_type=file_find.ANY) self.assertEqual(expected_files, actual_files)
def search(clazz, tarball, pattern, ignore_case=False, whole_word=False): 'Return the output of either ag (silver searcher) or grep for the contents of an archive.' ag = which.which('ag') if ag: cmd = [ag] else: grep = which.which('grep') if not grep: raise RuntimeError('No grep or ag found.') cmd = [grep, '-r'] if ignore_case: cmd.append('-i') if whole_word: cmd.append('-w') cmd.extend([pattern, '.']) tmp_dir = temp_file.make_temp_dir() archiver.extract(tarball, tmp_dir, strip_common_ancestor=True) result = execute.execute(cmd, cwd=tmp_dir, shell=True, raise_error=False) file_util.remove(tmp_dir) return result
def test_find_file_duplicates_with_setup_and_removed_resolved_file(self): items = [ temp_content('file', 'src/a/kiwi.jpg', 'this is kiwi', 0o0644), temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644), temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644), temp_content('file', 'src/b/kiwi_dup1.jpg', 'this is kiwi', 0o0644), temp_content('file', 'src/c/kiwi_dup2.jpg', 'this is kiwi', 0o0644), temp_content('file', 'foo/cheese/brie.jpg', 'this is kiwi', 0o0644), temp_content('file', 'foo/cheese/cheddar.jpg', 'this is cheddar', 0o0644), temp_content('file', 'foo/cheese/gouda.jpg', 'this is lemon', 0o0644), ] options = file_duplicates_options(recursive=True) with dir_operation_tester(extra_content_items=items) as t: setup = file_duplicates.setup([t.src_dir], options=options) file_util.remove(f'{t.src_dir}/a/kiwi.jpg') dups = file_duplicates.find_file_duplicates_with_setup( f'{t.tmp_dir}/foo/cheese/brie.jpg', setup) self.assertEqual([ f'{t.src_dir}/b/kiwi_dup1.jpg', f'{t.src_dir}/c/kiwi_dup2.jpg', ], dups) dups = file_duplicates.find_file_duplicates_with_setup( f'{t.tmp_dir}/foo/cheese/gouda.jpg', setup) self.assertEqual([ f'{t.src_dir}/a/lemon.jpg', ], dups)
def _delete_one(self): files = self._list_trash() self.log_i('_delete_one(files = %s)' % (files)) if not files: return False to_delete = files.pop(0) file_util.remove(to_delete) return len(files) > 0
def _local_checksum(self, filename): if self.compressed: tmp_uncompressed_file = temp_file.make_temp_file() compressed_file.uncompress(filename, tmp_uncompressed_file) result = file_util.checksum('sha256', tmp_uncompressed_file) file_util.remove(tmp_uncompressed_file) else: result = file_util.checksum('sha256', filename)
def test_set_value_non_existent_file(self): 'Set the first value for a non existent properties file.' tmp = temp_file.make_temp_file() file_util.remove(tmp) e = PE(tmp) e.set_value('fruit', 'kiwi') expected = """fruit: kiwi\n""" self.assertMultiLineEqual(expected, file_util.read(tmp, codec = 'utf-8') )
def _delete_one(self): files = self._list_trash() self.log_i('_delete_one(files = %s)' % (files)) if not files: return False to_delete = files.pop(0) file_util.remove(to_delete) return len(files) > 0
def test_set_value_non_existent_file(self): 'Set the first value for a non existent properties file.' tmp = self.make_temp_file(prefix='set_value_non_existent_file') file_util.remove(tmp) e = PE(tmp) e.set_value('fruit', 'kiwi') expected = """fruit: kiwi\n""" self.assert_text_file_equal(expected, tmp)
def commit(clazz, root_dir, message, filenames): filenames = object_util.listify(filenames) tmp_msg = temp_file.make_temp_file(content = message) args = [ 'commit', '-F', tmp_msg ] + filenames try: rv = git_exe.call_git(root_dir, args) finally: file_util.remove(tmp_msg) return clazz.last_commit_hash(root_dir, short_hash = True)
def repo_bump_tag(clazz, address, component, dry_run, reset_lower=False): 'Bump the tag of a repo by address.' repo = clazz.clone_to_temp_dir(address) result = repo.bump_tag(component, push=True, dry_run=dry_run, reset_lower=reset_lower) file_util.remove(repo.root) return result
def remove_members(clazz, archive, members, debug=False): 'Remove members from an archive and then recreate it.' members = object_util.listify(members) tmp_dir = archiver.extract_all_temp_dir(archive, delete=not debug) if debug: print('tmp_dir: {}'.format(tmp_dir)) members = [path.normpath(path.join(tmp_dir, m)) for m in members] file_util.remove(members) archiver.create(archive, tmp_dir)
def _collect_side_effects(table, test, where, label, keep_side_effects): droppings = file_find.find(where, relative=False, file_type=file_find.ANY) for next_dropping in droppings: if not test in table: table[test] = [] se = side_effect(where, next_dropping, label) table[test].append(se) if not keep_side_effects: file_util.remove(next_dropping)
def remove_stream(clazz, filename, stream_name): 'Remove stream_name from filename if it exists.' filename = file_check.check_file(filename) stream_name = clazz.check_stream_name(stream_name) if not clazz.has_stream(filename, stream_name): return ads_filename = clazz._make_ads_filename(filename, stream_name) file_util.remove(ads_filename)
def _test_many_concurrent_worker(clazz, fruit, address): tmp_dir = clazz.make_temp_dir() content = clazz._make_content(fruit) repo = git_repo(tmp_dir, address=address) repo.clone_or_pull() repo.add_file(fruit, content=fruit, commit=True) repo.push_with_rebase(num_tries=10, retry_wait_seconds=0.250) file_util.remove(tmp_dir) return 0
def test_resolve_cyclic_error(self): tmp1 = self.make_temp_file(suffix = '-one') tmp2 = self.make_temp_file(suffix = '-two') file_util.remove(tmp2) os.symlink(tmp1, tmp2) file_util.remove(tmp1) os.symlink(tmp2, tmp1) with self.assertRaises(IOError) as ctx: file_symlink.resolve(tmp1) self.assertTrue( 'Cyclic error' in str(ctx.exception) )
def test_set_value_non_existent_file(self): 'Set the first value for a non existent config file.' tmp = temp_file.make_temp_file() file_util.remove(tmp) e = SCE(tmp) e.set_value('something', 'fruit', 'kiwi') expected = '''\ something fruit: kiwi ''' self.assert_text_file_equal(expected, tmp, native_line_breaks=True)
def test_nonexistent_file(self): tmp_file = self.make_temp_file() file_util.remove(tmp_file) c = ssh_known_hosts_file(tmp_file) c.add_known_host(ssh_known_host([ 'foo.com', '192.168.2.2' ], 'ssh-rsa', 'key4')) expected = ''' foo.com,192.168.2.2 ssh-rsa key4 ''' self.assertMultiLineEqual( expected.strip(), file_util.read(tmp_file, codec = 'utf-8').strip() )
def _move_dir(clazz, from_dir, dest_dir): #print('FOO: from_dir: %s' % (from_dir)) #print('FOO: dest_dir: %s' % (dest_dir)) file_util.mkdir(dest_dir) # if file_util.same_device_id(from_dir, dest_dir): # print('FOO: calling shutil.move(%s, %s)' % (from_dir, dest_dir)) # assert False # shutil.move(from_dir, dest_dir) # return file_copy.copy_tree(from_dir, dest_dir) file_util.remove(from_dir)
def _cleanup_usr_local_links(self): 'Cleanup symlinks in /usr/local/bin that break after uninstalling python' links = dir_util.list('/usr/local/bin') broken_links = [l for l in links if file_symlink.is_broken(l)] for broken_link in broken_links: target = os.readlink(broken_link) if 'Library/Frameworks/Python.framework/Versions' in target: self.blurb_verbose( 'Removing broken /usr/local link: {}'.format(broken_link)) file_util.remove(broken_link)
def delete_file(self, filename): 'Delete a file or directory.' if path.isfile(filename): self.log_i('deleting single file %s' % (filename)) file_util.remove(filename) elif path.isdir(filename): files = dir_util.list(filename) self.log_i('deleting many files: %s' % (files)) for f in files: self.log_i('deleting next file %s' % (f)) file_util.remove(f) self.log_i('sleeping for %f seconds' % (self._sleep_time)) time.sleep(self._sleep_time) else: raise RuntimeError('invalid file type: %s' % (filename))
def delete_file(self, filename): 'Delete a file or directory.' self.log_i('deleting %s' % (filename)) file_util.remove(filename)