def test_download_with_auth(self): tester = file_web_server_tester(debug=self.DEBUG, users={'fred': 'flintpass'}) tester.write_temp_content([ 'file foo.txt "this is foo.txt\n"', 'file subdir/bar.txt "bar.txt\n"', 'file subdir/subberdir/baz.txt "this is baz.txt\n"', 'file emptyfile.txt', 'dir emptydir', ]) tester.start() url = tester.make_url('foo.txt') download_tmp = url_util.download_to_temp_file(url, auth=('fred', 'flintpass'), suffix='.txt') self.assertEqual('text/plain', file_mime.mime_type(download_tmp)) self.assertEqual('this is foo.txt\n', file_util.read(download_tmp, codec='utf8')) url = tester.make_url('subdir/subberdir/baz.txt') download_tmp = url_util.download_to_temp_file(url, auth=('fred', 'flintpass'), suffix='.txt') self.assertEqual('text/plain', file_mime.mime_type(download_tmp)) self.assertEqual('this is baz.txt\n', file_util.read(download_tmp, codec='utf8')) with self.assertRaises((url_compat.HTTPError, RuntimeError)) as ctx: url = tester.make_url('notthere.txt') tmp = url_util.download_to_temp_file(url) tester.stop()
def test_compressed_no_uncompress(self): tmp_dir = self._make_temp_content([ 'file foo.txt "this is foo.txt\n"', 'file subdir/bar.txt "bar.txt\n"', 'file subdir/subberdir/baz.txt "this is baz.txt\n"', 'file emptyfile.txt', 'dir emptydir', ]) server = web_server_controller(file_web_server) server.start(root_dir=tmp_dir) port = server.address[1] cache = http_download_cache(temp_file.make_temp_dir(), compressed=True) url1 = self._make_url(port, 'foo.txt') self.assertFalse(cache.has_url(url1)) self.assertEqual(0, cache.download_count) cached_filename = cache.get_url(url1, uncompress=False) tmp_uncompressed_file = self.make_temp_file() compressed_file.uncompress(cached_filename, tmp_uncompressed_file) self.assertEqual("this is foo.txt\n", file_util.read(tmp_uncompressed_file, codec='utf-8')) cached_filename = cache.get_url(url1, uncompress=False) tmp_uncompressed_file = self.make_temp_file() compressed_file.uncompress(cached_filename, tmp_uncompressed_file) self.assertEqual("this is foo.txt\n", file_util.read(tmp_uncompressed_file, codec='utf-8')) server.stop()
def vm_host_ssh_setup(self, ssh_config_dir, filename, username, builder_access_ssh_public_key, builder_access_ssh_private_key): 'Setup ssh config inside a vm host instance' check.check_string(ssh_config_dir) check.check_string(filename) check.check_string(username) check.check_string(builder_access_ssh_public_key) check.check_string(builder_access_ssh_private_key) self.log.log_d( 'vm_host_ssh_setup: ssh_config_dir={} filename={} username={}'. format(ssh_config_dir, filename, username)) cm = ssh_config_manager(ssh_config_dir) # Install inbound keys to ssh into the vm builder from the vm host builder_access_ssh_public_key_content = file_util.read( builder_access_ssh_public_key, codec='utf-8') builder_access_ssh_private_key_content = file_util.read( builder_access_ssh_private_key, codec='utf-8') installed = cm.install_key_pair( builder_access_ssh_public_key_content, builder_access_ssh_private_key_content, filename) self.log.log_d( 'vm_builder_ssh_setup: access installed={}'.format(installed)) return 0
def test_replace_value(self): 'Set the first value for a non existent properties file.' tmp = temp_file.make_temp_file(content = '') e = PE(tmp) e.set_value('fruit', 'kiwi') expected = """fruit: kiwi\n""" self.assertMultiLineEqual(expected, file_util.read(tmp, codec = 'utf-8') ) e.set_value('fruit', 'orange') expected = """fruit: orange\n""" self.assertMultiLineEqual(expected, file_util.read(tmp, codec = 'utf-8') )
def test_cached_filename(self): tmp_cache_dir = self.make_temp_dir(prefix='test_cached_root_', suffix='.dir') tmp_filename = self.make_temp_file(content='foo\n') #print('tmp_cache_dir={}'.format(tmp_cache_dir)) #print('tmp_filename={}'.format(tmp_filename)) expected_content = file_util.read(tmp_filename) cached_filename = file_cache.cached_filename(tmp_filename, cache_dir=tmp_cache_dir) actual_content = file_util.read(cached_filename) self.assertEqual(expected_content, actual_content) self.assertNotEqual(tmp_filename, cached_filename)
def test_set_value_existing_file(self): 'Add a second property to an existing property file.' content = """\ fruit: 'kiwi' """ tmp = temp_file.make_temp_file(content = content) e = PE(tmp) self.assertMultiLineEqual(content, file_util.read(tmp, codec = 'utf-8') ) e.set_value('status', 'doomed') expected = """\ fruit: kiwi status: doomed """ self.assertMultiLineEqual(expected, file_util.read(tmp, codec = 'utf-8') )
def test_basic(self): content = '''\ This is my nice file. ''' tmp_file = self.make_temp_file(content=content) tmp_compressed_file = self.make_temp_file() compressed_file.compress(tmp_file, tmp_compressed_file) tmp_uncompressed_file = self.make_temp_file() compressed_file.uncompress(tmp_compressed_file, tmp_uncompressed_file) self.assertMultiLineEqual( file_util.read(tmp_file, codec='utf-8'), file_util.read(tmp_uncompressed_file, codec='utf-8'))
def install_package(self, package_filename): 'Install a python package directly. Not always supported.' check.check_string(package_filename) self._log.log_method_d() log_dir = temp_file.make_temp_dir(prefix='python_install_', suffix='.dir', delete=not self.options.debug) install_log = path.join(log_dir, 'install.log') self._log.log_d('install_package: install_log={}'.format(install_log)) cmd = [ package_filename, '/log', install_log, '/quiet', '/silent', 'InstallAllUsers=1', 'PrependPath=0', 'Shortcuts=0', 'AssociateFiles=0', 'Include_doc=0', 'Include_launcher=0', 'InstallLauncherAllUsers=0', ] self._log.log_d('install_package: command={}'.format(' '.join(cmd))) rv = execute.execute(cmd, stderr_to_stdout=True, raise_error=False) self._log.log_d('install_package: exit_code={} output={}'.format( rv.exit_code, rv.stdout)) if rv.exit_code != 0: print(file_util.read(install_log, codec='utf-8'))
def test_update_config(self): content = '''\ [something] fruit = kiwi cheese = brie wine = barolo ''' tmp = temp_file.make_temp_file(content=content) e = CFE(tmp) e.update_config({ 'something': { 'fruit': 'lemon', 'candy': 'chocolate', }, 'foo': { 'bread': 'baguette', 'drink': 'water', }, }) expected = '''\ [something] fruit = lemon cheese = brie wine = barolo candy = chocolate [foo] bread = baguette drink = water ''' self.assertMultiLineEqual(expected, file_util.read(tmp, codec='utf-8'))
def test_import_file_clobber(self): content1 = '''\ [something] fruit = kiwi ''' content2 = '''\ [something] cheese = brie ''' content3 = '''\ [something] fruit = lemon ''' tmp = temp_file.make_temp_file() e = CFE(tmp) e.import_file(temp_file.make_temp_file(content=content1)) e.import_file(temp_file.make_temp_file(content=content2)) e.import_file(temp_file.make_temp_file(content=content3)) expected = '''\ [something] fruit = lemon cheese = brie ''' self.assertMultiLineEqual(expected, file_util.read(tmp, codec='utf-8'))
def test_import_file_empty_config(self): content1 = '''\ [something] fruit = kiwi ''' content2 = '''\ [something] cheese = brie ''' content3 = '''\ [something] wine = barolo ''' content4 = '''\ ''' tmp = temp_file.make_temp_file() e = CFE(tmp) e.import_file(temp_file.make_temp_file(content=content1)) e.import_file(temp_file.make_temp_file(content=content2)) e.import_file(temp_file.make_temp_file(content=content3)) e.import_file(temp_file.make_temp_file(content=content4)) expected = '''\ [something] fruit = kiwi cheese = brie wine = barolo ''' self.assertMultiLineEqual(expected, file_util.read(tmp, codec='utf-8'))
def test_remove_branch(self): text = '''\ [submodule "foo"] path = foo url = [email protected]:org/foo.git branch = b2 [submodule "bar"] path = bar url = [email protected]:org/bar.git branch = b666 ''' tmp_file = self.make_temp_file(content=text) mf = git_modules_file(tmp_file) mf.set_branch('foo', None) mf.set_branch('bar', None) expected = '''\ [submodule "foo"] path = foo url = [email protected]:org/foo.git [submodule "bar"] path = bar url = [email protected]:org/bar.git ''' self.assertMultiLineEqual(expected, file_util.read(tmp_file, codec='utf8'))
def run_test(self, config, function, function_args = None): tmp_output_filename = self.make_temp_file(suffix = '.output') p = self._log_tester(function, config, tmp_output_filename, function_args) p.start() p.join() output = file_util.read(tmp_output_filename, codec = 'utf-8') return self._test_result(tmp_output_filename, output)
def test_file_sync_change_one(self): tmp_src_dir1 = self._make_temp_content([ 'file foo.txt "first foo.txt\n"', 'file subdir/bar.txt "bar.txt\n"', 'file subdir/subberdir/baz.txt "baz.txt\n"', 'file emptyfile.txt', 'dir emptydir', ]) tmp_src_dir2 = self._make_temp_content([ 'file foo.txt "second foo.txt\n"', 'file subdir/bar.txt "bar.txt\n"', 'file subdir/subberdir/baz.txt "baz.txt\n"', 'file emptyfile.txt', 'dir emptydir', ]) tmp_dst_dir = temp_file.make_temp_dir() file_sync.sync(tmp_src_dir1, tmp_dst_dir) file_sync.sync(tmp_src_dir2, tmp_dst_dir) expected = [ self.native_filename('emptyfile.txt'), self.native_filename('foo.txt'), self.native_filename('subdir/bar.txt'), self.native_filename('subdir/subberdir/baz.txt'), ] self.assertEqual( expected, file_find.find(tmp_dst_dir, relative = True) ) self.assertEqual( 'second foo.txt\n', file_util.read(path.join(tmp_dst_dir, 'foo.txt'), codec = 'utf8') )
def handle_request(self, environ, start_response): path_info = environ['PATH_INFO'] self.log_i('handle_request(%s)' % (path_info)) if path_info not in self._known_tarballs: return self.response_error(start_response, 404) extension = file_util.extension(path_info) if 'large' in path_info: items = [ temp_archive.item('kiwi.bin', content=self._make_large_content()), ] else: items = [ temp_archive.item('apple.txt', content='apple.txt\n'), temp_archive.item('orange.txt', content='orange.txt\n'), ] tmp_archive = temp_archive.make_temp_archive(items, extension) tmp_mime_type = file_mime.mime_type(tmp_archive) content = file_util.read(tmp_archive) headers = [ ('Content-Type', str(tmp_mime_type)), ('Content-Length', str(len(content))), ] result = self.response_success(start_response, 200, [content], headers) file_util.remove(tmp_archive) return result
def test_basic(self): # This is the document to write to the repository. tmp_source_doc = self.make_temp_file(content='abc', suffix='-doc.txt') # Here's a repository to put it in. r = git_temp_repo(debug=self.DEBUG) r.add_file('dummy.txt', content='dummy') r.push('origin', 'master') tmp_db_dir = self.make_temp_dir() args = [ 'git_repo_document', 'update', tmp_source_doc, r.address, 'master', '--working-dir', tmp_db_dir, ] rv = self.run_program(self._program, args) self.assertEqual(0, rv.exit_code) # Also check doc is in repo. It will have the same name as the source doc. filename = path.basename(tmp_source_doc) r.pull() contents = r.read_file(filename) self.assertEqual(contents, 'abc') # Since we now have a file in the repo, let's also test whether load_document can read it. # Here's an auto-delete temp directory for the document DB's local repository. tmp_db_dir2 = temp_file.make_temp_dir(delete=not self.DEBUG) # By default, the CLI will put the output file in a file with the same name as the file in # the repo, in the current directory. tmp_target_filepath = path.join(getcwd(), filename) if path.exists(tmp_target_filepath): remove(tmp_target_filepath) # Run the CLI. args = [ 'git_repo_document', 'load', filename, r.address, 'master', '--working-dir', tmp_db_dir2, ] rv = self.run_program(self._program, args) self.assertEqual(0, rv.exit_code) # See if the file and contents are there. actual = file_util.read(tmp_target_filepath) self.assertEqual(actual, b'abc') # This will cause an exception and fail the test if the file wasn't created by the CLI. remove(tmp_target_filepath)
def test_replace_value_quoted(self): 'Set the first value for a non existent config file.' tmp = temp_file.make_temp_file() file_util.remove(tmp) e = CFE(tmp, string_quote_char='"') e.set_value('something', 'fruit', 'kiwi') expected = '''\ [something] fruit = "kiwi" ''' self.assertMultiLineEqual(expected, file_util.read(tmp, codec='utf-8')) e.set_value('something', 'fruit', 'apple') expected = '''\ [something] fruit = "apple" ''' self.assertMultiLineEqual(expected, file_util.read(tmp, codec='utf-8'))
def test_set_value_non_existent_file(self): 'Set the first value for a non existent properties file.' tmp = temp_file.make_temp_file() file_util.remove(tmp) e = PE(tmp) e.set_value('fruit', 'kiwi') expected = """fruit: kiwi\n""" self.assertMultiLineEqual(expected, file_util.read(tmp, codec = 'utf-8') )
def __new__(clazz, filename): filename = path.abspath(filename) check.check_string(filename) file_check.check_file(filename) content = file_util.read(filename, codec='utf-8') root_dir = path.normpath(path.join(path.dirname(filename), '..')) data = config_data.parse(content, filename=filename) return clazz.__bases__[0].__new__(clazz, root_dir, filename, data)
def download_to_bytes(self, remote_filename): 'Download filename to local_filename.' remote_filename = vfs_path_util.normalize(remote_filename) p = self._make_local_file_path(remote_filename) if not path.exists(p): raise vfs_error('file not found: {}'.format(remote_filename)) if not path.isfile(p): raise vfs_error('not a file: {}'.format(remote_filename)) return file_util.read(p)
def xtest_cached_content(self): tmp_cache_dir = self.make_temp_dir(prefix='test_cached_root_', suffix='.dir') tmp_filename = self.make_temp_file(content='foo\n') expected_content = file_util.read(tmp_filename) actual_content = file_cache.cached_content(tmp_filename, cache_dir=tmp_cache_dir) self.assertEqual(expected_content, actual_content)
def test_get_value_existing_file(self): content = '''\ [something] fruit = kiwi ''' tmp = temp_file.make_temp_file(content=content) e = CFE(tmp) self.assertMultiLineEqual(content, file_util.read(tmp, codec='utf-8')) self.assertEqual('kiwi', e.get_value('something', 'fruit'))
def test_get_bytes_with_change(self): tmp = self.make_temp_file(content='this is foo', suffix='.txt') yesterday = datetime.now() - timedelta(days=1) file_util.set_modification_date(tmp, yesterday) counter = 0 def _value_maker1(f): nonlocal counter counter += 1 return b'666' def _value_maker2(f): nonlocal counter counter += 1 return b'667' self.assertEqual(0, counter) self.assertEqual( b'666', file_attributes_metadata.get_bytes(tmp, 'foo', _value_maker1)) self.assertEqual(1, counter) self.assertEqual( b'666', file_attributes_metadata.get_bytes(tmp, 'foo', _value_maker1)) self.assertEqual(1, counter) mtime = file_util.get_modification_date(tmp) self.assertEqual( { '__bes_mtime_foo__': str(mtime.timestamp()).encode('utf-8'), 'foo': b'666', }, file_attributes.get_all(tmp)) with open(tmp, 'a') as f: f.write(' more text') f.flush() self.assertEqual('this is foo more text', file_util.read(tmp, codec='utf-8')) self.assertEqual( b'667', file_attributes_metadata.get_bytes(tmp, 'foo', _value_maker2)) self.assertEqual(2, counter) self.assertEqual( b'667', file_attributes_metadata.get_bytes(tmp, 'foo', _value_maker2)) self.assertEqual(2, counter) new_mtime = file_util.get_modification_date(tmp) self.assertEqual( { '__bes_mtime_foo__': str( new_mtime.timestamp()).encode('utf-8'), 'foo': b'667', }, file_attributes.get_all(tmp))
def test_install_key_pair_for_host(self): tmp_dir = self.make_temp_dir(suffix='.ssh') cm = ssh_config_manager(tmp_dir) tmp_public_key = 'publickey' tmp_private_key = 'privatekey' tmp_hostname = 'bitbucket.org' installed = cm.install_key_pair_for_host('publickey', 'privatekey', 'bitbucket.org', username='******', include_ip_address=False, include_comment=False) files = file_find.find(tmp_dir, relative=True) self.assertEqual([ 'config', 'id_rsa_bitbucket_org', 'id_rsa_bitbucket_org.pub', 'known_hosts', ], files) expected_config = ''' Host bitbucket.org Hostname bitbucket.org IdentityFile {private_key} User fred '''.format(private_key=installed.private_key_filename) self.assertMultiLineEqual( expected_config.strip(), file_util.read(path.join(tmp_dir, 'config'), codec='utf-8').strip()) expected_known_hosts = ''' bitbucket.org ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAubiN81eDcafrgMeLzaFPsw2kNvEcqTKl/VqLat/MaB33pZy0y3rJZtnqwR2qOOvbwKZYKiEO1O6VqNEBxKvJJelCq0dTXWT5pbO2gDXC6h6QDXCaHo6pOHGPUy+YBaGQRGuSusMEASYiWunYN0vCAI8QaXnWMXNMdFP3jHAJH0eDsoiGnLPBlBp4TNm6rYI74nMzgz3B9IikW4WVK+dc8KZJZWYjAuORU3jc1c/NPskD2ASinf8v3xnfXeukU0sJ5N6m5E8VLjObPEO+mN2t/FZTMZLiFqPWc/ALSqnMnnhwrNi2rbfg/rd/IpL8Le3pSBne8+seeFVBoGqzHM9yXw== ''' self.assertMultiLineEqual( expected_known_hosts.strip(), file_util.read(path.join(tmp_dir, 'known_hosts'), codec='utf-8').strip()) abs_files = [path.join(tmp_dir, f) for f in files] for filename in abs_files: self.assertEqual(0o0600, file_util.mode(filename))
def test_get_value_missing_key(self): content = '''\ [something] fruit = kiwi ''' tmp = temp_file.make_temp_file(content=content) e = CFE(tmp) self.assertMultiLineEqual(content, file_util.read(tmp, codec='utf-8')) with self.assertRaises(KeyError) as ctx: self.assertEqual(None, e.get_value('something', 'color'))
def __new__(clazz, arcname, content=None, filename=None): assert content or filename if content: assert not filename if filename: assert not content if filename: content = file_util.read(filename) return clazz.__bases__[0].__new__(clazz, arcname, content, filename)
def find_nodes(clazz, filename, node_type): check.check_refactor_ast_node_type(node_type) filename = file_check.check_file(filename) source_code = file_util.read(filename, codec='utf-8') tree = ast.parse(source_code) nodes = refactor_ast_util.find_nodes(tree, node_type) return nodes
def test_compressed(self): tmp_dir = self._make_temp_content([ 'file foo.txt "this is foo.txt\n"', 'file subdir/bar.txt "bar.txt\n"', 'file subdir/subberdir/baz.txt "this is baz.txt\n"', 'file emptyfile.txt', 'dir emptydir', ]) server = web_server_controller(file_web_server) server.start(root_dir=tmp_dir) port = server.address[1] cache = http_download_cache(temp_file.make_temp_dir(), compressed=True) url1 = self._make_url(port, 'foo.txt') self.assertFalse(cache.has_url(url1)) self.assertEqual(0, cache.download_count) self.assertEqual("this is foo.txt\n", file_util.read(cache.get_url(url1), codec='utf-8')) self.assertTrue(cache.has_url(url1)) self.assertEqual(1, cache.download_count) self.assertEqual("this is foo.txt\n", file_util.read(cache.get_url(url1), codec='utf-8')) self.assertEqual(1, cache.download_count) self.assertEqual([ 'http___localhost_{}_foo.txt.gz'.format(server.address[1]), ], cache.find_all_files(relative=True)) url2 = self._make_url(port, 'subdir/subberdir/baz.txt') self.assertFalse(cache.has_url(url2)) self.assertEqual("this is baz.txt\n", file_util.read(cache.get_url(url2), codec='utf-8')) self.assertEqual("this is baz.txt\n", file_util.read(cache.get_url(url2), codec='utf-8')) self.assertTrue(cache.has_url(url2)) self.assertEqual([ 'http___localhost_{}_foo.txt.gz'.format(port), 'http___localhost_{}_subdir_subberdir_baz.txt.gz'.format(port), ], cache.find_all_files(relative=True)) server.stop()
def read_tag_list_file(clazz, filename): 'Read a list of tags from a file. Each line should be one tag' check.check_string(filename) text = file_util.read(filename) lines = text_line_parser.parse_lines(text, remove_empties=True) result = [] for i, line in enumerate(lines): if line: result.append(line) return sorted(result)
def read_easy_install_pth(clazz, filename): content = file_util.read(filename).strip() lines = content.decode('utf-8').split('\n') if len(lines) < 2: raise RuntimeError('Invalid easy-install.pth(1): %s' % (filename)) if not lines[0].startswith('import sys'): raise RuntimeError('Invalid easy-install.pth(2): %s' % (filename)) if not lines[-1].startswith('import sys'): raise RuntimeError('Invalid easy-install.pth(3): %s' % (filename)) eggs = lines[1:-1] return [path.normpath(egg) for egg in eggs]