def test_resolve_one_level(self): tmp1 = self.make_temp_file(suffix = '-one') tmp2 = self.make_temp_file(suffix = '-two') file_util.remove(tmp2) os.symlink(tmp1, tmp2) self.assertEqual( tmp1, file_symlink.resolve(tmp1) ) self.assertEqual( tmp1, file_symlink.resolve(tmp2) )
def test_is_broken_false(self): tmp1 = tempfile.NamedTemporaryFile() tmp2 = tempfile.NamedTemporaryFile() file_util.remove(tmp1.name) os.symlink(tmp2.name, tmp1.name) self.assertEqual( True, path.islink(tmp1.name) ) self.assertEqual( False, file_symlink.is_broken(tmp1.name) )
def run_script(clazz, exe, script, args=None):
'Run the script and return the result.'
check.check_string(exe)
check.check_string(script)
check.check_string_seq(args, allow_none=True)
args = list(args or [])
tmp_script = temp_file.make_temp_file(content=script, suffix='.py')
cmd = [exe, tmp_script] + args
clazz._log.log_d('run_script: exe={} cmd={}'.format(exe, cmd))
clazz._log.log_d('run_script: script=\n{}\n'.format(script))
try:
output_bytes = subprocess.check_output(cmd,
stderr=subprocess.STDOUT)
exit_code = 0
clazz._log.log_d('run_script: success')
except subprocess.CalledProcessError as ex:
clazz._log.log_d('run_script: caught: {}'.format(str(ex)))
output_bytes = ex.output
exit_code = ex.returncode
clazz._log.log_d('run_script: failed')
finally:
file_util.remove(tmp_script)
output = codecs.decode(output_bytes, 'utf-8').strip()
clazz._log.log_d('run_script: exit_code={} output="{}"')
return clazz._run_script_result(exit_code, output)
def _do_set_credentials(clazz, username, password, print_error):
expect_script = clazz._EXPECT_SCRIPT_TEMPLATE.format(username = username,
password = password)
# Remove ~/.vmrestCfg if corrupt
clazz._vmrest_config_remove_corrupt()
# Make sure there no stale vmrest processes
vmware_util.killall_vmrest()
tmp_script = temp_file.make_temp_file(suffix = '.expect', perm = 0o755, content = expect_script, delete = False)
try:
cmd = [ 'expect', '-d', tmp_script ]
rv = execute.execute(cmd, raise_error = False, shell = False)
clazz._log.log_d('_do_set_credentials: exit_code={} stdout="{}" stderr="{}"'.format(rv.exit_code,
rv.stdout,
rv.stderr))
if rv.exit_code != 0:
raise vmware_error(clazz._EXPECT_SCRIPT_MSG_MAP[rv.exit_code])
finally:
file_util.remove(tmp_script)
if clazz._vmrest_config_is_corrupt():
msg = 'Failed to set credentials. Corrupt {}'.format(clazz._VMREST_CONFIG_FILENAME)
if print_error:
clazz._log.log_i(msg)
raise vmware_error(msg)
def test_check_dir_symlink_success(self): tmp_dir = self.make_temp_dir() d = self.make_temp_dir(dir = tmp_dir) l = self.make_temp_file(content = 'link', dir = tmp_dir) file_symlink.symlink(d, l) self.assertEqual( d, file_check.check_dir(l) ) file_util.remove(tmp_dir)
def handle_request(self, environ, start_response):
path_info = environ['PATH_INFO']
self.log_i('handle_request(%s)' % (path_info))
if path_info not in self._known_tarballs:
return self.response_error(start_response, 404)
extension = file_util.extension(path_info)
if 'large' in path_info:
items = [
temp_archive.item('kiwi.bin',
content=self._make_large_content()),
]
else:
items = [
temp_archive.item('apple.txt', content='apple.txt\n'),
temp_archive.item('orange.txt', content='orange.txt\n'),
]
tmp_archive = temp_archive.make_temp_archive(items, extension)
tmp_mime_type = file_mime.mime_type(tmp_archive)
content = file_util.read(tmp_archive)
headers = [
('Content-Type', str(tmp_mime_type)),
('Content-Length', str(len(content))),
]
result = self.response_success(start_response, 200, [content],
headers)
file_util.remove(tmp_archive)
return result
def __exit__(self, type, value, traceback):
if not path.isdir(self._where):
raise IOError('Directory not found: "{}"'.format(self._where))
after = set(dir_util.list(self._where))
diff = after - self._before
if diff:
file_util.remove(list(diff))
def test_check_file_symlink_broken(self):
f = self.make_temp_file(content = 'kiwi')
l = self.make_temp_file(content = 'link')
file_symlink.symlink(f, l)
file_util.remove(f)
with self.assertRaises(IOError) as ctx:
file_check.check_file(l)
def backup(self, output_archive):
'Backup image to an gzipped tarball.'
tmp_tar = temp_file.make_temp_file(suffix='.tar')
args = ['save', self.tagged_repository, '-o', tmp_tar]
docker_exe.call_docker(args, non_blocking=False)
compressed_file.compress(tmp_tar, output_archive)
file_util.remove(tmp_tar)
def repo_run_operation(clazz,
address,
operation,
commit_message,
options=None):
check.check_string(address)
check.check_callable(operation)
check.check_string(commit_message)
check.check_git_repo_operation_options(options, allow_none=True)
options = options or git_repo_operation_options()
if options.verbose:
print('repo_run_operation: cloning {}'.format(address))
repo = clazz.clone_to_temp_dir(address,
options=options,
debug=options.debug)
if options.debug:
msg = 'repo_run_operation: repo.root={}'.format(repo.root)
print(msg)
clazz._LOG.log_d(msg)
if options.dry_run:
print('DRY_RUN: would execute {} on {} in {}'.format(
operation, address, repo.root))
else:
if options.verbose:
print('repo_run_operation: executing {} in cwd={}'.format(
operation, repo.root))
repo.operation_with_reset(
operation,
commit_message,
num_tries=options.num_tries,
retry_wait_seconds=options.retry_wait_seconds,
files_to_commit=options.files_to_commit)
file_util.remove(repo.root)
def _do_download_with_temp_ssh(clazz, address, revision, output_filename,
base_name, download_options, parsed_address,
ssh_credentials):
domain_name = parsed_address.service
clazz._log.log_d('_do_download_with_temp_ssh: domain_name={}'.format(domain_name))
temp_home = temp_file.make_temp_dir(delete = not download_options.debug)
if download_options.debug:
print('temp_home={}'.format(temp_home))
temp_ssh_dir = path.join(temp_home, '.ssh')
clazz._log.log_d('_do_download_with_temp_ssh: temp_ssh_dir={}'.format(temp_ssh_dir))
cm = ssh_config_manager(temp_ssh_dir)
installed = cm.install_key_pair_for_host(ssh_credentials.public_key,
ssh_credentials.private_key,
domain_name,
username = ssh_credentials.username,
host_key_type = ssh_credentials.host_key_type,
include_ip_address = True,
include_comment = True)
# use a temporary HOME with custom git and ssh config just for this download
with env_override.temp_home(use_temp_home = temp_home) as env:
git_config.set_identity('test', '*****@*****.**')
ssh_config_file = path.join(temp_ssh_dir, 'config')
ssh_command = 'ssh -F {}'.format(ssh_config_file)
git_config.set_value('core.sshCommand', ssh_command)
clazz._do_download(address, revision, output_filename, base_name, download_options)
file_util.remove(temp_home)
def test_check_dir_symlink_broken(self):
d = self.make_temp_dir()
l = self.make_temp_file(content = 'link')
file_symlink.symlink(d, l)
file_util.remove(d)
with self.assertRaises(IOError) as ctx:
file_check.check_dir(l)
def test_copy_tree_spaces_in_filenames(self):
self.maxDiff = None
src_tmp_dir = self.make_temp_dir(prefix='src-',
suffix='-has 2 spaces-')
dst_tmp_dir = self.make_temp_dir(prefix='dst-',
suffix='-has 2 spaces-')
file_util.remove(dst_tmp_dir)
with tarfile.open(self.data_path('test.tar'), mode='r') as f:
f.extractall(path=src_tmp_dir)
file_copy.copy_tree(src_tmp_dir, dst_tmp_dir)
expected_files = [
self.native_filename('1'),
self.native_filename('1/2'),
self.native_filename('1/2/3'),
self.native_filename('1/2/3/4'),
self.native_filename('1/2/3/4/5'),
self.native_filename('1/2/3/4/5/apple.txt'),
self.native_filename('1/2/3/4/5/kiwi.txt'),
self.native_filename('bar.txt'),
self.native_filename('empty'),
self.native_filename('foo.txt'),
self.native_filename('kiwi_link.txt'),
]
actual_files = file_find.find(dst_tmp_dir, file_type=file_find.ANY)
self.assertEqual(expected_files, actual_files)
def search(clazz, tarball, pattern, ignore_case=False, whole_word=False):
'Return the output of either ag (silver searcher) or grep for the contents of an archive.'
ag = which.which('ag')
if ag:
cmd = [ag]
else:
grep = which.which('grep')
if not grep:
raise RuntimeError('No grep or ag found.')
cmd = [grep, '-r']
if ignore_case:
cmd.append('-i')
if whole_word:
cmd.append('-w')
cmd.extend([pattern, '.'])
tmp_dir = temp_file.make_temp_dir()
archiver.extract(tarball, tmp_dir, strip_common_ancestor=True)
result = execute.execute(cmd,
cwd=tmp_dir,
shell=True,
raise_error=False)
file_util.remove(tmp_dir)
return result
def test_find_file_duplicates_with_setup_and_removed_resolved_file(self):
items = [
temp_content('file', 'src/a/kiwi.jpg', 'this is kiwi', 0o0644),
temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644),
temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644),
temp_content('file', 'src/b/kiwi_dup1.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/c/kiwi_dup2.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'foo/cheese/brie.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'foo/cheese/cheddar.jpg', 'this is cheddar',
0o0644),
temp_content('file', 'foo/cheese/gouda.jpg', 'this is lemon',
0o0644),
]
options = file_duplicates_options(recursive=True)
with dir_operation_tester(extra_content_items=items) as t:
setup = file_duplicates.setup([t.src_dir], options=options)
file_util.remove(f'{t.src_dir}/a/kiwi.jpg')
dups = file_duplicates.find_file_duplicates_with_setup(
f'{t.tmp_dir}/foo/cheese/brie.jpg', setup)
self.assertEqual([
f'{t.src_dir}/b/kiwi_dup1.jpg',
f'{t.src_dir}/c/kiwi_dup2.jpg',
], dups)
dups = file_duplicates.find_file_duplicates_with_setup(
f'{t.tmp_dir}/foo/cheese/gouda.jpg', setup)
self.assertEqual([
f'{t.src_dir}/a/lemon.jpg',
], dups)
def _delete_one(self):
files = self._list_trash()
self.log_i('_delete_one(files = %s)' % (files))
if not files:
return False
to_delete = files.pop(0)
file_util.remove(to_delete)
return len(files) > 0
def _local_checksum(self, filename):
if self.compressed:
tmp_uncompressed_file = temp_file.make_temp_file()
compressed_file.uncompress(filename, tmp_uncompressed_file)
result = file_util.checksum('sha256', tmp_uncompressed_file)
file_util.remove(tmp_uncompressed_file)
else:
result = file_util.checksum('sha256', filename)
def test_set_value_non_existent_file(self):
'Set the first value for a non existent properties file.'
tmp = temp_file.make_temp_file()
file_util.remove(tmp)
e = PE(tmp)
e.set_value('fruit', 'kiwi')
expected = """fruit: kiwi\n"""
self.assertMultiLineEqual(expected, file_util.read(tmp, codec = 'utf-8') )
def _delete_one(self):
files = self._list_trash()
self.log_i('_delete_one(files = %s)' % (files))
if not files:
return False
to_delete = files.pop(0)
file_util.remove(to_delete)
return len(files) > 0
def test_set_value_non_existent_file(self):
'Set the first value for a non existent properties file.'
tmp = self.make_temp_file(prefix='set_value_non_existent_file')
file_util.remove(tmp)
e = PE(tmp)
e.set_value('fruit', 'kiwi')
expected = """fruit: kiwi\n"""
self.assert_text_file_equal(expected, tmp)
def commit(clazz, root_dir, message, filenames):
filenames = object_util.listify(filenames)
tmp_msg = temp_file.make_temp_file(content = message)
args = [ 'commit', '-F', tmp_msg ] + filenames
try:
rv = git_exe.call_git(root_dir, args)
finally:
file_util.remove(tmp_msg)
return clazz.last_commit_hash(root_dir, short_hash = True)
def repo_bump_tag(clazz, address, component, dry_run, reset_lower=False):
'Bump the tag of a repo by address.'
repo = clazz.clone_to_temp_dir(address)
result = repo.bump_tag(component,
push=True,
dry_run=dry_run,
reset_lower=reset_lower)
file_util.remove(repo.root)
return result
def remove_members(clazz, archive, members, debug=False):
'Remove members from an archive and then recreate it.'
members = object_util.listify(members)
tmp_dir = archiver.extract_all_temp_dir(archive, delete=not debug)
if debug:
print('tmp_dir: {}'.format(tmp_dir))
members = [path.normpath(path.join(tmp_dir, m)) for m in members]
file_util.remove(members)
archiver.create(archive, tmp_dir)
def _collect_side_effects(table, test, where, label, keep_side_effects):
droppings = file_find.find(where, relative=False, file_type=file_find.ANY)
for next_dropping in droppings:
if not test in table:
table[test] = []
se = side_effect(where, next_dropping, label)
table[test].append(se)
if not keep_side_effects:
file_util.remove(next_dropping)
def remove_stream(clazz, filename, stream_name):
'Remove stream_name from filename if it exists.'
filename = file_check.check_file(filename)
stream_name = clazz.check_stream_name(stream_name)
if not clazz.has_stream(filename, stream_name):
return
ads_filename = clazz._make_ads_filename(filename, stream_name)
file_util.remove(ads_filename)
def _test_many_concurrent_worker(clazz, fruit, address):
tmp_dir = clazz.make_temp_dir()
content = clazz._make_content(fruit)
repo = git_repo(tmp_dir, address=address)
repo.clone_or_pull()
repo.add_file(fruit, content=fruit, commit=True)
repo.push_with_rebase(num_tries=10, retry_wait_seconds=0.250)
file_util.remove(tmp_dir)
return 0
def test_resolve_cyclic_error(self):
tmp1 = self.make_temp_file(suffix = '-one')
tmp2 = self.make_temp_file(suffix = '-two')
file_util.remove(tmp2)
os.symlink(tmp1, tmp2)
file_util.remove(tmp1)
os.symlink(tmp2, tmp1)
with self.assertRaises(IOError) as ctx:
file_symlink.resolve(tmp1)
self.assertTrue( 'Cyclic error' in str(ctx.exception) )
def test_set_value_non_existent_file(self):
'Set the first value for a non existent config file.'
tmp = temp_file.make_temp_file()
file_util.remove(tmp)
e = SCE(tmp)
e.set_value('something', 'fruit', 'kiwi')
expected = '''\
something
fruit: kiwi
'''
self.assert_text_file_equal(expected, tmp, native_line_breaks=True)
def test_nonexistent_file(self):
tmp_file = self.make_temp_file()
file_util.remove(tmp_file)
c = ssh_known_hosts_file(tmp_file)
c.add_known_host(ssh_known_host([ 'foo.com', '192.168.2.2' ], 'ssh-rsa', 'key4'))
expected = '''
foo.com,192.168.2.2 ssh-rsa key4
'''
self.assertMultiLineEqual( expected.strip(), file_util.read(tmp_file, codec = 'utf-8').strip() )
def _move_dir(clazz, from_dir, dest_dir):
#print('FOO: from_dir: %s' % (from_dir))
#print('FOO: dest_dir: %s' % (dest_dir))
file_util.mkdir(dest_dir)
# if file_util.same_device_id(from_dir, dest_dir):
# print('FOO: calling shutil.move(%s, %s)' % (from_dir, dest_dir))
# assert False
# shutil.move(from_dir, dest_dir)
# return
file_copy.copy_tree(from_dir, dest_dir)
file_util.remove(from_dir)
def _cleanup_usr_local_links(self):
'Cleanup symlinks in /usr/local/bin that break after uninstalling python'
links = dir_util.list('/usr/local/bin')
broken_links = [l for l in links if file_symlink.is_broken(l)]
for broken_link in broken_links:
target = os.readlink(broken_link)
if 'Library/Frameworks/Python.framework/Versions' in target:
self.blurb_verbose(
'Removing broken /usr/local link: {}'.format(broken_link))
file_util.remove(broken_link)
def delete_file(self, filename):
'Delete a file or directory.'
if path.isfile(filename):
self.log_i('deleting single file %s' % (filename))
file_util.remove(filename)
elif path.isdir(filename):
files = dir_util.list(filename)
self.log_i('deleting many files: %s' % (files))
for f in files:
self.log_i('deleting next file %s' % (f))
file_util.remove(f)
self.log_i('sleeping for %f seconds' % (self._sleep_time))
time.sleep(self._sleep_time)
else:
raise RuntimeError('invalid file type: %s' % (filename))
def delete_file(self, filename):
'Delete a file or directory.'
self.log_i('deleting %s' % (filename))
file_util.remove(filename)
