def test_file_or_dir(self): tmp_file = temp_file.make_temp_dir() tmp_dir = temp_file.make_temp_dir() self.assertTrue( file_type.matches(tmp_file, file_type.DIR | file_type.FILE)) self.assertTrue( file_type.matches(tmp_dir, file_type.DIR | file_type.FILE))
def _make_context(self, niceness_level=None, timeout=None, deleter=None): tmp_dir = temp_file.make_temp_dir() trash_dir = path.join(tmp_dir, 'trash') stuff_dir = path.join(tmp_dir, 'stuff') trash = file_trash(temp_file.make_temp_dir(), niceness_level=niceness_level, timeout=timeout, deleter=deleter) return self._context(trash_dir, stuff_dir, trash)
def install_package(self, package_filename): 'Install a python package directly. Not always supported.' check.check_string(package_filename) self._log.log_method_d() log_dir = temp_file.make_temp_dir(prefix='python_install_', suffix='.dir', delete=not self.options.debug) install_log = path.join(log_dir, 'install.log') self._log.log_d('install_package: install_log={}'.format(install_log)) cmd = [ package_filename, '/log', install_log, '/quiet', '/silent', 'InstallAllUsers=1', 'PrependPath=0', 'Shortcuts=0', 'AssociateFiles=0', 'Include_doc=0', 'Include_launcher=0', 'InstallLauncherAllUsers=0', ] self._log.log_d('install_package: command={}'.format(' '.join(cmd))) rv = execute.execute(cmd, stderr_to_stdout=True, raise_error=False) self._log.log_d('install_package: exit_code={} output={}'.format( rv.exit_code, rv.stdout)) if rv.exit_code != 0: print(file_util.read(install_log, codec='utf-8'))
def __init__(self, root_dir = None, debug = False, items = None, users = None): self.root_dir = root_dir or temp_file.make_temp_dir(delete = not debug) if items: self.write_temp_content(items) self.server = None self.port = None self.users = users
def archive(clazz, address, revision, base_name, output_filename, untracked = False, override_gitignore = None, debug = False): 'git archive with additional support to include untracked files for local repos.' tmp_repo_dir = temp_file.make_temp_dir(delete = not debug) if path.isdir(address): excludes = git_ignore.read_gitignore_file(address) file_copy.copy_tree(address, tmp_repo_dir, excludes = excludes) if override_gitignore: file_util.save(path.join(address, '.gitignore'), content = override_gitignore) if untracked: git_exe.call_git(tmp_repo_dir, [ 'add', '-A' ]) git_exe.call_git(tmp_repo_dir, [ 'commit', '-m', '"add untracked files just for tmp repo"' ]) else: if untracked: raise git_error('untracked can only be True for local repos.') clazz.clone(address, tmp_repo_dir) output_filename = path.abspath(output_filename) file_util.mkdir(path.dirname(output_filename)) args = [ 'archive', '--format=tgz', '--prefix=%s-%s/' % (base_name, revision), '-o', output_filename, revision ] rv = git_exe.call_git(tmp_repo_dir, args) return rv
def _init_remote(self, content, prefix, commit_message=None): if prefix: remote_prefix = '{}remote-'.format(prefix) else: remote_prefix = 'remote-' self._remote_repo = self._make_temp_repo( init_args=['--bare', '--shared'], debug=self._debug, prefix=remote_prefix, commit_message=commit_message) if prefix: local_prefix = '{}local-'.format(prefix) else: local_prefix = 'local-' tmp_dir = temp_file.make_temp_dir(delete=not self._debug, prefix=local_prefix) if self._debug: print('git_temp_repo: tmp_dir: %s' % (tmp_dir)) self._local_repo = git_repo(tmp_dir, address=self._remote_repo.root) self._local_repo.clone() if content: self._local_repo.write_temp_content(items=content, commit=True, commit_message=commit_message) self._local_repo.push('origin', 'master') self.root = self._local_repo.root self.address = self._remote_repo.root self._transplant_methods(self._local_repo)
def _do_download_with_temp_ssh(clazz, address, revision, output_filename, base_name, download_options, parsed_address, ssh_credentials): domain_name = parsed_address.service clazz._log.log_d('_do_download_with_temp_ssh: domain_name={}'.format(domain_name)) temp_home = temp_file.make_temp_dir(delete = not download_options.debug) if download_options.debug: print('temp_home={}'.format(temp_home)) temp_ssh_dir = path.join(temp_home, '.ssh') clazz._log.log_d('_do_download_with_temp_ssh: temp_ssh_dir={}'.format(temp_ssh_dir)) cm = ssh_config_manager(temp_ssh_dir) installed = cm.install_key_pair_for_host(ssh_credentials.public_key, ssh_credentials.private_key, domain_name, username = ssh_credentials.username, host_key_type = ssh_credentials.host_key_type, include_ip_address = True, include_comment = True) # use a temporary HOME with custom git and ssh config just for this download with env_override.temp_home(use_temp_home = temp_home) as env: git_config.set_identity('test', '*****@*****.**') ssh_config_file = path.join(temp_ssh_dir, 'config') ssh_command = 'ssh -F {}'.format(ssh_config_file) git_config.set_value('core.sshCommand', ssh_command) clazz._do_download(address, revision, output_filename, base_name, download_options) file_util.remove(temp_home)
def _downlod_url(clazz, url, debug=False): if not url_util.exists(url): raise python_installer_error( 'No python.org package found: "{}"'.format(url)) tmp_dir = temp_file.make_temp_dir(suffix='-python-download', delete=not debug) basename = path.basename(url) tmp_package = path.join(tmp_dir, basename) url_util.download_to_file(url, tmp_package) if debug: print('tmp python package download: {}'.format(tmp_package)) expected_checksum = clazz._fetch_checksum(url) if not expected_checksum: raise python_installer_error( 'Failed to determine checksum for: {}'.format(url)) actual_checksum = file_util.checksum('md5', tmp_package) if expected_checksum != actual_checksum: msg = ''' CHECKSUM MISMATCH: url={url} CHECKSUM MISMATCH: expected={expected} CHECKSUM MISMATCH: actual={actual} CHECKSUM MISMATCH: filename={filename} CHECKSUM MISMATCH: run with --debug to keep and debug the download '''.format(url=url, expected=expected_checksum, actual=actual_checksum, filename=tmp_package) raise python_installer_error(msg) return tmp_package
def test_file_sync_change_one(self): tmp_src_dir1 = self._make_temp_content([ 'file foo.txt "first foo.txt\n"', 'file subdir/bar.txt "bar.txt\n"', 'file subdir/subberdir/baz.txt "baz.txt\n"', 'file emptyfile.txt', 'dir emptydir', ]) tmp_src_dir2 = self._make_temp_content([ 'file foo.txt "second foo.txt\n"', 'file subdir/bar.txt "bar.txt\n"', 'file subdir/subberdir/baz.txt "baz.txt\n"', 'file emptyfile.txt', 'dir emptydir', ]) tmp_dst_dir = temp_file.make_temp_dir() file_sync.sync(tmp_src_dir1, tmp_dst_dir) file_sync.sync(tmp_src_dir2, tmp_dst_dir) expected = [ self.native_filename('emptyfile.txt'), self.native_filename('foo.txt'), self.native_filename('subdir/bar.txt'), self.native_filename('subdir/subberdir/baz.txt'), ] self.assertEqual( expected, file_find.find(tmp_dst_dir, relative = True) ) self.assertEqual( 'second foo.txt\n', file_util.read(path.join(tmp_dst_dir, 'foo.txt'), codec = 'utf8') )
def test_compressed_no_uncompress(self): tmp_dir = self._make_temp_content([ 'file foo.txt "this is foo.txt\n"', 'file subdir/bar.txt "bar.txt\n"', 'file subdir/subberdir/baz.txt "this is baz.txt\n"', 'file emptyfile.txt', 'dir emptydir', ]) server = web_server_controller(file_web_server) server.start(root_dir=tmp_dir) port = server.address[1] cache = http_download_cache(temp_file.make_temp_dir(), compressed=True) url1 = self._make_url(port, 'foo.txt') self.assertFalse(cache.has_url(url1)) self.assertEqual(0, cache.download_count) cached_filename = cache.get_url(url1, uncompress=False) tmp_uncompressed_file = self.make_temp_file() compressed_file.uncompress(cached_filename, tmp_uncompressed_file) self.assertEqual("this is foo.txt\n", file_util.read(tmp_uncompressed_file, codec='utf-8')) cached_filename = cache.get_url(url1, uncompress=False) tmp_uncompressed_file = self.make_temp_file() compressed_file.uncompress(cached_filename, tmp_uncompressed_file) self.assertEqual("this is foo.txt\n", file_util.read(tmp_uncompressed_file, codec='utf-8')) server.stop()
def lfs_invalid_files(clazz, address, branch): ''' Return a list of files that are GIT LFS invalid. That means: - They match the .gitattributes - They are not GIT LFS pointers but plain git files ''' check.check_string(address) check.check_string(branch) '''\ Cloning into 'lfs-test'... remote: Counting objects: 96, done. remote: Compressing objects: 100% (89/89), done. remote: Total 96 (delta 18), reused 0 (delta 0) Receiving objects: 100% (96/96), 76.11 KiB | 1.14 MiB/s, done. Resolving deltas: 100% (18/18), done. Filtering content: 100% (2/2), 20.00 KiB | 7.00 KiB/s, done. Encountered 3 file(s) that should have been pointers, but weren't: honeydew melon.bytes lemon.bytes subdir/organic banana.bytes ''' tmp_dir = temp_file.make_temp_dir() repo = git_repo(tmp_dir, address=address) options = git_clone_options() options.branch = branch clone_rv, sub_rv = repo.clone(options=options) print('clone_rv: {}'.format(clone_rv)) print(' sub_rv: {}'.format(sub_rv)) return []
def search(clazz, tarball, pattern, ignore_case=False, whole_word=False): 'Return the output of either ag (silver searcher) or grep for the contents of an archive.' ag = which.which('ag') if ag: cmd = [ag] else: grep = which.which('grep') if not grep: raise RuntimeError('No grep or ag found.') cmd = [grep, '-r'] if ignore_case: cmd.append('-i') if whole_word: cmd.append('-w') cmd.extend([pattern, '.']) tmp_dir = temp_file.make_temp_dir() archiver.extract(tarball, tmp_dir, strip_common_ancestor=True) result = execute.execute(cmd, cwd=tmp_dir, shell=True, raise_error=False) file_util.remove(tmp_dir) return result
def test_basic(self): # This is the document to write to the repository. tmp_source_doc = self.make_temp_file(content='abc', suffix='-doc.txt') # Here's a repository to put it in. r = git_temp_repo(debug=self.DEBUG) r.add_file('dummy.txt', content='dummy') r.push('origin', 'master') tmp_db_dir = self.make_temp_dir() args = [ 'git_repo_document', 'update', tmp_source_doc, r.address, 'master', '--working-dir', tmp_db_dir, ] rv = self.run_program(self._program, args) self.assertEqual(0, rv.exit_code) # Also check doc is in repo. It will have the same name as the source doc. filename = path.basename(tmp_source_doc) r.pull() contents = r.read_file(filename) self.assertEqual(contents, 'abc') # Since we now have a file in the repo, let's also test whether load_document can read it. # Here's an auto-delete temp directory for the document DB's local repository. tmp_db_dir2 = temp_file.make_temp_dir(delete=not self.DEBUG) # By default, the CLI will put the output file in a file with the same name as the file in # the repo, in the current directory. tmp_target_filepath = path.join(getcwd(), filename) if path.exists(tmp_target_filepath): remove(tmp_target_filepath) # Run the CLI. args = [ 'git_repo_document', 'load', filename, r.address, 'master', '--working-dir', tmp_db_dir2, ] rv = self.run_program(self._program, args) self.assertEqual(0, rv.exit_code) # See if the file and contents are there. actual = file_util.read(tmp_target_filepath) self.assertEqual(actual, b'abc') # This will cause an exception and fail the test if the file wasn't created by the CLI. remove(tmp_target_filepath)
def recreate_temp_file(clazz, archive, base_dir, delete=True): 'Recreate the archive to a temp file.' tmp_dir = temp_file.make_temp_dir(delete=True) clazz.extract_all(archive, tmp_dir) return clazz.create_temp_file( archive_extension.extension_for_filename(archive), tmp_dir, base_dir=base_dir)
def make_temp_cloned_repo(self, prefix=None): tmp_dir = temp_file.make_temp_dir(delete=not self._debug, prefix=prefix) if self._debug: print('git_temp_repo: tmp_dir: %s' % (tmp_dir)) r = git_repo(tmp_dir, address=self._remote_repo.root) r.clone() return r
def _tmp_example_dmg(self, filename): tmp_dir = temp_file.make_temp_dir(delete=not self.DEBUG) tmp_example = path.join(tmp_dir, filename) if self.DEBUG: print('tmp_example: %s' % (tmp_example)) file_util.copy(self.data_path(filename), tmp_example, use_hard_link=True) return tmp_example
def _do_download(clazz, address, revision, output_filename, base_name, download_options): clazz._log.log_d('_do_download: home={} real_home={}'.format(path.expanduser('~'), user.HOME)) tmp_root_dir = temp_file.make_temp_dir(delete = not download_options.debug) if download_options.debug: print('tmp_root_dir={}'.format(tmp_root_dir)) repo = git_repo(tmp_root_dir, address = address) repo.clone(options = download_options) repo.archive_to_file(base_name, revision, output_filename, archive_format = 'tgz')
def test_file_sync_with_mode(self): tmp_src_dir = self._make_temp_content([ 'file foo.txt "foo.txt\n" 644', 'file bar.sh "#!/bin/bash\necho bar\n" 755', ]) tmp_dst_dir = temp_file.make_temp_dir() file_sync.sync(tmp_src_dir, tmp_dst_dir) self.assertEqual( 0o0755, file_util.mode(path.join(tmp_dst_dir, 'bar.sh')) ) self.assertEqual( 0o0644, file_util.mode(path.join(tmp_dst_dir, 'foo.txt')) )
def test_extract_all(self): tmp_dir = temp_file.make_temp_dir() info1 = dmg.info() dmg.extract(self._tmp_example_dmg('example.dmg'), tmp_dir) info2 = dmg.info() files = file_find.find(tmp_dir, relative=True, file_type=file_find.FILE_OR_LINK) self.assertEqual(['foo.txt', 'link_to_foo.sh', 'subdir/bar.txt'], files)
def clone_to_temp_dir(clazz, address, options=None, debug=False): 'Clone a git address to a temp dir' tmp_dir = temp_file.make_temp_dir(delete=not debug) clazz._LOG.log_d('clone_to_temp_dir: tmp_dir={}'.format(tmp_dir)) if debug: print('clone_to_temp_dir: tmp_dir={}'.format(tmp_dir)) repo = git_repo(tmp_dir, address=address) repo.clone(options=options) assert repo.root == tmp_dir return repo
def extract_member_to_file(self, member, filename): tmp_dir = temp_file.make_temp_dir() tmp_member = path.join(tmp_dir, member) self.extract(tmp_dir, include=[member]) if not path.exists(tmp_member): raise RuntimeError('Failed to extract member from {}: {}'.format( self.filename, member)) if not path.isfile(tmp_member): raise RuntimeError('Member is not a file {}: {}'.format( self.filename, member)) file_util.rename(tmp_member, filename)
def _make_temp_archive_dmg(clazz, items, filename, mode): tmp_dir = temp_file.make_temp_dir() for item in items: assert item assert item.arcname file_util.save(path.join(tmp_dir, item.arcname), content=item.content) tmp_dmg = temp_file.make_temp_file() cmd = 'hdiutil create -srcfolder %s -ov -format UDZO %s' % (tmp_dir, filename) execute.execute(cmd) file_util.remove(tmp_dir)
def _make_temp_archive_xz(clazz, items, filename, mode): tmp_dir = temp_file.make_temp_dir() for item in items: assert item assert item.arcname file_util.save(path.join(tmp_dir, item.arcname), content=item.content) tmp_xz = temp_file.make_temp_file() manifest_content = '\n'.join([item.arcname for item in items]) manifest = temp_file.make_temp_file(content=manifest_content) cmd = 'tar Jcf %s -C %s -T %s' % (filename, tmp_dir, manifest) execute.execute(cmd) file_util.remove(tmp_dir)
def extract_all_temp_dir(clazz, filename, base_dir=None, strip_common_ancestor=False, strip_head=None, delete=True): tmp_dir = temp_file.make_temp_dir(delete=delete) clazz.extract_all(filename, tmp_dir, base_dir=base_dir, strip_common_ancestor=strip_common_ancestor, strip_head=strip_head) return tmp_dir
def run(self): args = self.parser.parse_args() blurb.set_verbose(args.verbose) args.python_versions = dim_python.resolve_python_versions( args.python_versions) args.systems = dim_system.resolve_systems(args.systems) args.steps = dim_step.resolve_steps(args.steps) self.logs_dir = path.join(os.getcwd(), 'BUILD', 'logs') self.egoist = self._find_egoist() self.temp_dir = temp_file.make_temp_dir(suffix='build') self.repo = git_repo('.', find_root=True) if args.print_ecr_images: self._print_ecr_images() return 0 self.bes_version = args.bes_version or self.repo.greatest_remote_tag() self.blurb(' steps: {}'.format(' '.join(args.steps))) self.blurb(' systems: {}'.format(' '.join(args.systems))) self.blurb(' python_versions: {}'.format(' '.join( args.python_versions))) self.blurb(' build: {}'.format(args.build)) self.blurb(' test: {}'.format(args.test)) self.blurb(' publish_egoist: {}'.format(args.publish_egoist)) self.blurb(' publish_image: {}'.format(args.publish_image)) self.blurb(' clean: {}'.format(args.clean)) self.blurb(' very_clean: {}'.format(args.very_clean)) self.blurb('bes_version: {}'.format(self.bes_version)) if args.clean and args.very_clean: self.blurb('Only one of --clean or --very-clean should be given.') return 1 if args.clean: self._clean() return 0 if args.very_clean: self._very_clean() return 0 result = self._do_run(args) if result.success: self.blurb('SUCCESS') return 0 for failed_entry in result.failed_entries: self.blurb('FAILED: {} => {}'.format( failed_entry.descriptor.docker_tag, failed_entry.log)) return 1
def diff_contents(clazz, archive1, archive2, strip_common_ancestor=False): 'Return the output of diffing the contents of 2 archives.' tmp_dir = temp_file.make_temp_dir(delete=True) tmp_dir1 = path.join(tmp_dir, 'a') tmp_dir2 = path.join(tmp_dir, 'b') archiver.extract_all(archive1, tmp_dir1, strip_common_ancestor=strip_common_ancestor) archiver.extract_all(archive2, tmp_dir2, strip_common_ancestor=strip_common_ancestor) cmd = ['diff', '-u', '-r', tmp_dir1, tmp_dir2] rv = execute.execute(cmd, raise_error=False, stderr_to_stdout=True) return rv
def test_update_egg_directory(self): tmp_dir = temp_file.make_temp_dir() eggs = [ 'foo-1.2.3-py2.7.egg', 'bar-6.6.6-py2.7.egg', 'baz-10.11.12-py2.7.egg', ] for egg in eggs: file_util.save(path.join(tmp_dir, egg), content = '%s\n' % (egg)) setup_tools.update_egg_directory(tmp_dir) easy_install_dot_pth_path = path.join(tmp_dir, setup_tools.EASY_INSTALL_DOT_PTH_FILENAME) actual_eggs = setup_tools.read_easy_install_pth(easy_install_dot_pth_path) self.assertEqual( sorted(eggs), sorted(actual_eggs) )
def _init_local(self, content, prefix, commit_message=None): tmp_dir = temp_file.make_temp_dir(delete=not self._debug, prefix=prefix) if self._debug: print('git_temp_repo: tmp_dir: %s' % (tmp_dir)) self._local_repo = git_repo(tmp_dir, address=None) self._local_repo.init() if content: self._local_repo.write_temp_content(items=content, commit=True, commit_message=commit_message) self.root = self._local_repo.root self.address = None self._transplant_methods(self._local_repo)
def archive_to_file(clazz, root, prefix, revision, output_filename, archive_format = None, short_hash = True): 'git archive to a archive file.' prefix = file_util.ensure_rsep(prefix) archive_format = archive_format or 'tgz' output_filename = path.abspath(output_filename) file_util.ensure_file_dir(output_filename) if short_hash: if clazz.is_long_hash(revision): revision = clazz.short_hash(root, revision) clazz.log.log_d('archive_to_file: revision={} output_filename={} archive_format={}'.format(revision, output_filename, archive_format)) tmp_dir = temp_file.make_temp_dir() clazz.archive_to_dir(root, revision, tmp_dir) archiver.create(output_filename, tmp_dir, base_dir = prefix, extension = archive_format)
def uninstall(self, version): '''Uninstall a python by version or full_version.' Uninstall a python version using any of these forms: major.minor.revision major.minor ''' version = python_version.check_version_or_full_version(version) installed_versions = self.installed_versions() self._log.log_d('uninstall: installed_versions: {}'.format( installed_versions.to_string())) matching_versions = installed_versions.filter_by_version(version) matching_versions.sort() self._log.log_d('uninstall: matching_versions: {}'.format( matching_versions.to_string())) if not matching_versions: raise python_installer_error('Not installed: {}'.format(version)) if len(matching_versions) != 1: raise python_installer_error( 'Somehow multiple installed versions found for {}: {}'.format( versiom, matching_versions.to_string())) full_version = matching_versions[0] self._log.log_d('uninstall: full_version: {}'.format(full_version)) assert full_version old_package = self.download(full_version) self._log.log_d('uninstall: old_package={}'.format(old_package)) log_dir = temp_file.make_temp_dir(prefix='python_install_', suffix='.dir', delete=not self.options.debug) uninstall_log = path.join(log_dir, 'uninstall.log') self._log.log_d('uninstall: uninstall_log={}'.format(uninstall_log)) cmd = [ old_package, '/uninstall', '/log', uninstall_log, '/quiet', '/silent', ] self._log.log_d('uninstall: command={}'.format(' '.join(cmd))) rv = execute.execute(cmd, stderr_to_stdout=True, raise_error=False) self._log.log_d('uninstall: exit_code={} output={}'.format( rv.exit_code, rv.stdout)) if rv.exit_code != 0: print(file_util.read(uninstall_log, codec='utf-8'))