def test_download_with_auth(self):
tester = file_web_server_tester(debug=self.DEBUG,
users={'fred': 'flintpass'})
tester.write_temp_content([
'file foo.txt "this is foo.txt\n"',
'file subdir/bar.txt "bar.txt\n"',
'file subdir/subberdir/baz.txt "this is baz.txt\n"',
'file emptyfile.txt',
'dir emptydir',
])
tester.start()
url = tester.make_url('foo.txt')
download_tmp = url_util.download_to_temp_file(url,
auth=('fred',
'flintpass'),
suffix='.txt')
self.assertEqual('text/plain', file_mime.mime_type(download_tmp))
self.assertEqual('this is foo.txt\n',
file_util.read(download_tmp, codec='utf8'))
url = tester.make_url('subdir/subberdir/baz.txt')
download_tmp = url_util.download_to_temp_file(url,
auth=('fred',
'flintpass'),
suffix='.txt')
self.assertEqual('text/plain', file_mime.mime_type(download_tmp))
self.assertEqual('this is baz.txt\n',
file_util.read(download_tmp, codec='utf8'))
with self.assertRaises((url_compat.HTTPError, RuntimeError)) as ctx:
url = tester.make_url('notthere.txt')
tmp = url_util.download_to_temp_file(url)
tester.stop()
def test_compressed_no_uncompress(self):
tmp_dir = self._make_temp_content([
'file foo.txt "this is foo.txt\n"',
'file subdir/bar.txt "bar.txt\n"',
'file subdir/subberdir/baz.txt "this is baz.txt\n"',
'file emptyfile.txt',
'dir emptydir',
])
server = web_server_controller(file_web_server)
server.start(root_dir=tmp_dir)
port = server.address[1]
cache = http_download_cache(temp_file.make_temp_dir(), compressed=True)
url1 = self._make_url(port, 'foo.txt')
self.assertFalse(cache.has_url(url1))
self.assertEqual(0, cache.download_count)
cached_filename = cache.get_url(url1, uncompress=False)
tmp_uncompressed_file = self.make_temp_file()
compressed_file.uncompress(cached_filename, tmp_uncompressed_file)
self.assertEqual("this is foo.txt\n",
file_util.read(tmp_uncompressed_file, codec='utf-8'))
cached_filename = cache.get_url(url1, uncompress=False)
tmp_uncompressed_file = self.make_temp_file()
compressed_file.uncompress(cached_filename, tmp_uncompressed_file)
self.assertEqual("this is foo.txt\n",
file_util.read(tmp_uncompressed_file, codec='utf-8'))
server.stop()
def vm_host_ssh_setup(self, ssh_config_dir, filename, username,
builder_access_ssh_public_key,
builder_access_ssh_private_key):
'Setup ssh config inside a vm host instance'
check.check_string(ssh_config_dir)
check.check_string(filename)
check.check_string(username)
check.check_string(builder_access_ssh_public_key)
check.check_string(builder_access_ssh_private_key)
self.log.log_d(
'vm_host_ssh_setup: ssh_config_dir={} filename={} username={}'.
format(ssh_config_dir, filename, username))
cm = ssh_config_manager(ssh_config_dir)
# Install inbound keys to ssh into the vm builder from the vm host
builder_access_ssh_public_key_content = file_util.read(
builder_access_ssh_public_key, codec='utf-8')
builder_access_ssh_private_key_content = file_util.read(
builder_access_ssh_private_key, codec='utf-8')
installed = cm.install_key_pair(
builder_access_ssh_public_key_content,
builder_access_ssh_private_key_content, filename)
self.log.log_d(
'vm_builder_ssh_setup: access installed={}'.format(installed))
return 0
def test_replace_value(self):
'Set the first value for a non existent properties file.'
tmp = temp_file.make_temp_file(content = '')
e = PE(tmp)
e.set_value('fruit', 'kiwi')
expected = """fruit: kiwi\n"""
self.assertMultiLineEqual(expected, file_util.read(tmp, codec = 'utf-8') )
e.set_value('fruit', 'orange')
expected = """fruit: orange\n"""
self.assertMultiLineEqual(expected, file_util.read(tmp, codec = 'utf-8') )
def test_cached_filename(self):
tmp_cache_dir = self.make_temp_dir(prefix='test_cached_root_',
suffix='.dir')
tmp_filename = self.make_temp_file(content='foo\n')
#print('tmp_cache_dir={}'.format(tmp_cache_dir))
#print('tmp_filename={}'.format(tmp_filename))
expected_content = file_util.read(tmp_filename)
cached_filename = file_cache.cached_filename(tmp_filename,
cache_dir=tmp_cache_dir)
actual_content = file_util.read(cached_filename)
self.assertEqual(expected_content, actual_content)
self.assertNotEqual(tmp_filename, cached_filename)
def test_set_value_existing_file(self):
'Add a second property to an existing property file.'
content = """\
fruit: 'kiwi'
"""
tmp = temp_file.make_temp_file(content = content)
e = PE(tmp)
self.assertMultiLineEqual(content, file_util.read(tmp, codec = 'utf-8') )
e.set_value('status', 'doomed')
expected = """\
fruit: kiwi
status: doomed
"""
self.assertMultiLineEqual(expected, file_util.read(tmp, codec = 'utf-8') )
def test_basic(self):
content = '''\
This is my nice file.
'''
tmp_file = self.make_temp_file(content=content)
tmp_compressed_file = self.make_temp_file()
compressed_file.compress(tmp_file, tmp_compressed_file)
tmp_uncompressed_file = self.make_temp_file()
compressed_file.uncompress(tmp_compressed_file, tmp_uncompressed_file)
self.assertMultiLineEqual(
file_util.read(tmp_file, codec='utf-8'),
file_util.read(tmp_uncompressed_file, codec='utf-8'))
def install_package(self, package_filename):
'Install a python package directly. Not always supported.'
check.check_string(package_filename)
self._log.log_method_d()
log_dir = temp_file.make_temp_dir(prefix='python_install_',
suffix='.dir',
delete=not self.options.debug)
install_log = path.join(log_dir, 'install.log')
self._log.log_d('install_package: install_log={}'.format(install_log))
cmd = [
package_filename,
'/log',
install_log,
'/quiet',
'/silent',
'InstallAllUsers=1',
'PrependPath=0',
'Shortcuts=0',
'AssociateFiles=0',
'Include_doc=0',
'Include_launcher=0',
'InstallLauncherAllUsers=0',
]
self._log.log_d('install_package: command={}'.format(' '.join(cmd)))
rv = execute.execute(cmd, stderr_to_stdout=True, raise_error=False)
self._log.log_d('install_package: exit_code={} output={}'.format(
rv.exit_code, rv.stdout))
if rv.exit_code != 0:
print(file_util.read(install_log, codec='utf-8'))
def test_update_config(self):
content = '''\
[something]
fruit = kiwi
cheese = brie
wine = barolo
'''
tmp = temp_file.make_temp_file(content=content)
e = CFE(tmp)
e.update_config({
'something': {
'fruit': 'lemon',
'candy': 'chocolate',
},
'foo': {
'bread': 'baguette',
'drink': 'water',
},
})
expected = '''\
[something]
fruit = lemon
cheese = brie
wine = barolo
candy = chocolate
[foo]
bread = baguette
drink = water
'''
self.assertMultiLineEqual(expected, file_util.read(tmp, codec='utf-8'))
def test_import_file_clobber(self):
content1 = '''\
[something]
fruit = kiwi
'''
content2 = '''\
[something]
cheese = brie
'''
content3 = '''\
[something]
fruit = lemon
'''
tmp = temp_file.make_temp_file()
e = CFE(tmp)
e.import_file(temp_file.make_temp_file(content=content1))
e.import_file(temp_file.make_temp_file(content=content2))
e.import_file(temp_file.make_temp_file(content=content3))
expected = '''\
[something]
fruit = lemon
cheese = brie
'''
self.assertMultiLineEqual(expected, file_util.read(tmp, codec='utf-8'))
def test_import_file_empty_config(self):
content1 = '''\
[something]
fruit = kiwi
'''
content2 = '''\
[something]
cheese = brie
'''
content3 = '''\
[something]
wine = barolo
'''
content4 = '''\
'''
tmp = temp_file.make_temp_file()
e = CFE(tmp)
e.import_file(temp_file.make_temp_file(content=content1))
e.import_file(temp_file.make_temp_file(content=content2))
e.import_file(temp_file.make_temp_file(content=content3))
e.import_file(temp_file.make_temp_file(content=content4))
expected = '''\
[something]
fruit = kiwi
cheese = brie
wine = barolo
'''
self.assertMultiLineEqual(expected, file_util.read(tmp, codec='utf-8'))
def test_remove_branch(self):
text = '''\
[submodule "foo"]
path = foo
url = [email protected]:org/foo.git
branch = b2
[submodule "bar"]
path = bar
url = [email protected]:org/bar.git
branch = b666
'''
tmp_file = self.make_temp_file(content=text)
mf = git_modules_file(tmp_file)
mf.set_branch('foo', None)
mf.set_branch('bar', None)
expected = '''\
[submodule "foo"]
path = foo
url = [email protected]:org/foo.git
[submodule "bar"]
path = bar
url = [email protected]:org/bar.git
'''
self.assertMultiLineEqual(expected,
file_util.read(tmp_file, codec='utf8'))
def run_test(self, config, function, function_args = None): tmp_output_filename = self.make_temp_file(suffix = '.output') p = self._log_tester(function, config, tmp_output_filename, function_args) p.start() p.join() output = file_util.read(tmp_output_filename, codec = 'utf-8') return self._test_result(tmp_output_filename, output)
def test_file_sync_change_one(self):
tmp_src_dir1 = self._make_temp_content([
'file foo.txt "first foo.txt\n"',
'file subdir/bar.txt "bar.txt\n"',
'file subdir/subberdir/baz.txt "baz.txt\n"',
'file emptyfile.txt',
'dir emptydir',
])
tmp_src_dir2 = self._make_temp_content([
'file foo.txt "second foo.txt\n"',
'file subdir/bar.txt "bar.txt\n"',
'file subdir/subberdir/baz.txt "baz.txt\n"',
'file emptyfile.txt',
'dir emptydir',
])
tmp_dst_dir = temp_file.make_temp_dir()
file_sync.sync(tmp_src_dir1, tmp_dst_dir)
file_sync.sync(tmp_src_dir2, tmp_dst_dir)
expected = [
self.native_filename('emptyfile.txt'),
self.native_filename('foo.txt'),
self.native_filename('subdir/bar.txt'),
self.native_filename('subdir/subberdir/baz.txt'),
]
self.assertEqual( expected, file_find.find(tmp_dst_dir, relative = True) )
self.assertEqual( 'second foo.txt\n', file_util.read(path.join(tmp_dst_dir, 'foo.txt'), codec = 'utf8') )
def handle_request(self, environ, start_response):
path_info = environ['PATH_INFO']
self.log_i('handle_request(%s)' % (path_info))
if path_info not in self._known_tarballs:
return self.response_error(start_response, 404)
extension = file_util.extension(path_info)
if 'large' in path_info:
items = [
temp_archive.item('kiwi.bin',
content=self._make_large_content()),
]
else:
items = [
temp_archive.item('apple.txt', content='apple.txt\n'),
temp_archive.item('orange.txt', content='orange.txt\n'),
]
tmp_archive = temp_archive.make_temp_archive(items, extension)
tmp_mime_type = file_mime.mime_type(tmp_archive)
content = file_util.read(tmp_archive)
headers = [
('Content-Type', str(tmp_mime_type)),
('Content-Length', str(len(content))),
]
result = self.response_success(start_response, 200, [content],
headers)
file_util.remove(tmp_archive)
return result
def test_basic(self):
# This is the document to write to the repository.
tmp_source_doc = self.make_temp_file(content='abc', suffix='-doc.txt')
# Here's a repository to put it in.
r = git_temp_repo(debug=self.DEBUG)
r.add_file('dummy.txt', content='dummy')
r.push('origin', 'master')
tmp_db_dir = self.make_temp_dir()
args = [
'git_repo_document',
'update',
tmp_source_doc,
r.address,
'master',
'--working-dir',
tmp_db_dir,
]
rv = self.run_program(self._program, args)
self.assertEqual(0, rv.exit_code)
# Also check doc is in repo. It will have the same name as the source doc.
filename = path.basename(tmp_source_doc)
r.pull()
contents = r.read_file(filename)
self.assertEqual(contents, 'abc')
# Since we now have a file in the repo, let's also test whether load_document can read it.
# Here's an auto-delete temp directory for the document DB's local repository.
tmp_db_dir2 = temp_file.make_temp_dir(delete=not self.DEBUG)
# By default, the CLI will put the output file in a file with the same name as the file in
# the repo, in the current directory.
tmp_target_filepath = path.join(getcwd(), filename)
if path.exists(tmp_target_filepath):
remove(tmp_target_filepath)
# Run the CLI.
args = [
'git_repo_document',
'load',
filename,
r.address,
'master',
'--working-dir',
tmp_db_dir2,
]
rv = self.run_program(self._program, args)
self.assertEqual(0, rv.exit_code)
# See if the file and contents are there.
actual = file_util.read(tmp_target_filepath)
self.assertEqual(actual, b'abc')
# This will cause an exception and fail the test if the file wasn't created by the CLI.
remove(tmp_target_filepath)
def test_replace_value_quoted(self):
'Set the first value for a non existent config file.'
tmp = temp_file.make_temp_file()
file_util.remove(tmp)
e = CFE(tmp, string_quote_char='"')
e.set_value('something', 'fruit', 'kiwi')
expected = '''\
[something]
fruit = "kiwi"
'''
self.assertMultiLineEqual(expected, file_util.read(tmp, codec='utf-8'))
e.set_value('something', 'fruit', 'apple')
expected = '''\
[something]
fruit = "apple"
'''
self.assertMultiLineEqual(expected, file_util.read(tmp, codec='utf-8'))
def test_set_value_non_existent_file(self):
'Set the first value for a non existent properties file.'
tmp = temp_file.make_temp_file()
file_util.remove(tmp)
e = PE(tmp)
e.set_value('fruit', 'kiwi')
expected = """fruit: kiwi\n"""
self.assertMultiLineEqual(expected, file_util.read(tmp, codec = 'utf-8') )
def __new__(clazz, filename):
filename = path.abspath(filename)
check.check_string(filename)
file_check.check_file(filename)
content = file_util.read(filename, codec='utf-8')
root_dir = path.normpath(path.join(path.dirname(filename), '..'))
data = config_data.parse(content, filename=filename)
return clazz.__bases__[0].__new__(clazz, root_dir, filename, data)
def download_to_bytes(self, remote_filename):
'Download filename to local_filename.'
remote_filename = vfs_path_util.normalize(remote_filename)
p = self._make_local_file_path(remote_filename)
if not path.exists(p):
raise vfs_error('file not found: {}'.format(remote_filename))
if not path.isfile(p):
raise vfs_error('not a file: {}'.format(remote_filename))
return file_util.read(p)
def xtest_cached_content(self):
tmp_cache_dir = self.make_temp_dir(prefix='test_cached_root_',
suffix='.dir')
tmp_filename = self.make_temp_file(content='foo\n')
expected_content = file_util.read(tmp_filename)
actual_content = file_cache.cached_content(tmp_filename,
cache_dir=tmp_cache_dir)
self.assertEqual(expected_content, actual_content)
def test_get_value_existing_file(self):
content = '''\
[something]
fruit = kiwi
'''
tmp = temp_file.make_temp_file(content=content)
e = CFE(tmp)
self.assertMultiLineEqual(content, file_util.read(tmp, codec='utf-8'))
self.assertEqual('kiwi', e.get_value('something', 'fruit'))
def test_get_bytes_with_change(self):
tmp = self.make_temp_file(content='this is foo', suffix='.txt')
yesterday = datetime.now() - timedelta(days=1)
file_util.set_modification_date(tmp, yesterday)
counter = 0
def _value_maker1(f):
nonlocal counter
counter += 1
return b'666'
def _value_maker2(f):
nonlocal counter
counter += 1
return b'667'
self.assertEqual(0, counter)
self.assertEqual(
b'666',
file_attributes_metadata.get_bytes(tmp, 'foo', _value_maker1))
self.assertEqual(1, counter)
self.assertEqual(
b'666',
file_attributes_metadata.get_bytes(tmp, 'foo', _value_maker1))
self.assertEqual(1, counter)
mtime = file_util.get_modification_date(tmp)
self.assertEqual(
{
'__bes_mtime_foo__': str(mtime.timestamp()).encode('utf-8'),
'foo': b'666',
}, file_attributes.get_all(tmp))
with open(tmp, 'a') as f:
f.write(' more text')
f.flush()
self.assertEqual('this is foo more text',
file_util.read(tmp, codec='utf-8'))
self.assertEqual(
b'667',
file_attributes_metadata.get_bytes(tmp, 'foo', _value_maker2))
self.assertEqual(2, counter)
self.assertEqual(
b'667',
file_attributes_metadata.get_bytes(tmp, 'foo', _value_maker2))
self.assertEqual(2, counter)
new_mtime = file_util.get_modification_date(tmp)
self.assertEqual(
{
'__bes_mtime_foo__': str(
new_mtime.timestamp()).encode('utf-8'),
'foo': b'667',
}, file_attributes.get_all(tmp))
def test_install_key_pair_for_host(self):
tmp_dir = self.make_temp_dir(suffix='.ssh')
cm = ssh_config_manager(tmp_dir)
tmp_public_key = 'publickey'
tmp_private_key = 'privatekey'
tmp_hostname = 'bitbucket.org'
installed = cm.install_key_pair_for_host('publickey',
'privatekey',
'bitbucket.org',
username='******',
include_ip_address=False,
include_comment=False)
files = file_find.find(tmp_dir, relative=True)
self.assertEqual([
'config',
'id_rsa_bitbucket_org',
'id_rsa_bitbucket_org.pub',
'known_hosts',
], files)
expected_config = '''
Host bitbucket.org
Hostname bitbucket.org
IdentityFile {private_key}
User fred
'''.format(private_key=installed.private_key_filename)
self.assertMultiLineEqual(
expected_config.strip(),
file_util.read(path.join(tmp_dir, 'config'),
codec='utf-8').strip())
expected_known_hosts = '''
bitbucket.org ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAubiN81eDcafrgMeLzaFPsw2kNvEcqTKl/VqLat/MaB33pZy0y3rJZtnqwR2qOOvbwKZYKiEO1O6VqNEBxKvJJelCq0dTXWT5pbO2gDXC6h6QDXCaHo6pOHGPUy+YBaGQRGuSusMEASYiWunYN0vCAI8QaXnWMXNMdFP3jHAJH0eDsoiGnLPBlBp4TNm6rYI74nMzgz3B9IikW4WVK+dc8KZJZWYjAuORU3jc1c/NPskD2ASinf8v3xnfXeukU0sJ5N6m5E8VLjObPEO+mN2t/FZTMZLiFqPWc/ALSqnMnnhwrNi2rbfg/rd/IpL8Le3pSBne8+seeFVBoGqzHM9yXw==
'''
self.assertMultiLineEqual(
expected_known_hosts.strip(),
file_util.read(path.join(tmp_dir, 'known_hosts'),
codec='utf-8').strip())
abs_files = [path.join(tmp_dir, f) for f in files]
for filename in abs_files:
self.assertEqual(0o0600, file_util.mode(filename))
def test_get_value_missing_key(self):
content = '''\
[something]
fruit = kiwi
'''
tmp = temp_file.make_temp_file(content=content)
e = CFE(tmp)
self.assertMultiLineEqual(content, file_util.read(tmp, codec='utf-8'))
with self.assertRaises(KeyError) as ctx:
self.assertEqual(None, e.get_value('something', 'color'))
def __new__(clazz, arcname, content=None, filename=None):
assert content or filename
if content:
assert not filename
if filename:
assert not content
if filename:
content = file_util.read(filename)
return clazz.__bases__[0].__new__(clazz, arcname, content,
filename)
def find_nodes(clazz, filename, node_type):
check.check_refactor_ast_node_type(node_type)
filename = file_check.check_file(filename)
source_code = file_util.read(filename, codec='utf-8')
tree = ast.parse(source_code)
nodes = refactor_ast_util.find_nodes(tree, node_type)
return nodes
def test_compressed(self):
tmp_dir = self._make_temp_content([
'file foo.txt "this is foo.txt\n"',
'file subdir/bar.txt "bar.txt\n"',
'file subdir/subberdir/baz.txt "this is baz.txt\n"',
'file emptyfile.txt',
'dir emptydir',
])
server = web_server_controller(file_web_server)
server.start(root_dir=tmp_dir)
port = server.address[1]
cache = http_download_cache(temp_file.make_temp_dir(), compressed=True)
url1 = self._make_url(port, 'foo.txt')
self.assertFalse(cache.has_url(url1))
self.assertEqual(0, cache.download_count)
self.assertEqual("this is foo.txt\n",
file_util.read(cache.get_url(url1), codec='utf-8'))
self.assertTrue(cache.has_url(url1))
self.assertEqual(1, cache.download_count)
self.assertEqual("this is foo.txt\n",
file_util.read(cache.get_url(url1), codec='utf-8'))
self.assertEqual(1, cache.download_count)
self.assertEqual([
'http___localhost_{}_foo.txt.gz'.format(server.address[1]),
], cache.find_all_files(relative=True))
url2 = self._make_url(port, 'subdir/subberdir/baz.txt')
self.assertFalse(cache.has_url(url2))
self.assertEqual("this is baz.txt\n",
file_util.read(cache.get_url(url2), codec='utf-8'))
self.assertEqual("this is baz.txt\n",
file_util.read(cache.get_url(url2), codec='utf-8'))
self.assertTrue(cache.has_url(url2))
self.assertEqual([
'http___localhost_{}_foo.txt.gz'.format(port),
'http___localhost_{}_subdir_subberdir_baz.txt.gz'.format(port),
], cache.find_all_files(relative=True))
server.stop()
def read_tag_list_file(clazz, filename):
'Read a list of tags from a file. Each line should be one tag'
check.check_string(filename)
text = file_util.read(filename)
lines = text_line_parser.parse_lines(text, remove_empties=True)
result = []
for i, line in enumerate(lines):
if line:
result.append(line)
return sorted(result)
def read_easy_install_pth(clazz, filename):
content = file_util.read(filename).strip()
lines = content.decode('utf-8').split('\n')
if len(lines) < 2:
raise RuntimeError('Invalid easy-install.pth(1): %s' % (filename))
if not lines[0].startswith('import sys'):
raise RuntimeError('Invalid easy-install.pth(2): %s' % (filename))
if not lines[-1].startswith('import sys'):
raise RuntimeError('Invalid easy-install.pth(3): %s' % (filename))
eggs = lines[1:-1]
return [path.normpath(egg) for egg in eggs]